[flink] branch master updated (260ef2c -> 2aacb62)

[pn...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

pnowojski pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from 260ef2c  [FLINK-17634][rest] Reject multiple registration for the same endpoint
     add 8548d37  [FLINK-17547][task][hotfix] Improve error handling 1 catch one more invalid input in DataOutputSerializer.write 2 more informative error messages
     add 6e3c5ab  [FLINK-17547][task][hotfix] Extract NonSpanningWrapper from SpillingAdaptiveSpanningRecordDeserializer (static inner class) As it is, no logical changes.
     add d7b29f7  [FLINK-17547][task][hotfix] Extract SpanningWrapper from SpillingAdaptiveSpanningRecordDeserializer (static inner class). As it is, no logical changes.
     add 67d3eae  [FLINK-17547][task][hotfix] Fix compiler warnings in NonSpanningWrapper
     add 824100e  [FLINK-17547][task][hotfix] Extract methods from RecordsDeserializer
     add 37f441a  [FLINK-17547][task] Use iterator for unconsumed buffers. Motivation: support spilled records Changes: 1. change SpillingAdaptiveSpanningRecordDeserializer.getUnconsumedBuffer signature 2. adapt channel state persistence to new types
     add 179de29  [FLINK-17547][task][hotfix] Extract RefCountedFileWithStream from RefCountedFile Motivation: use RefCountedFile for reading as well.
     add 2fcc1fc  [FLINK-17547][task][hotfix] Move RefCountedFile to flink-core to use it in SpanningWrapper
     add 5415574  [FLINK-17547][task] Use RefCountedFile in SpanningWrapper (todo: merge with next?)
     add 2aacb62  [FLINK-17547][task] Implement getUnconsumedSegment for spilled buffers

No new revisions were added by this update.

Summary of changes:
 .../org/apache/flink/core/fs}/RefCountedFile.java  |  61 +-
 .../flink/core/memory/DataOutputSerializer.java    |   4 +-
 .../flink/core/memory/HybridMemorySegment.java     |   3 +-
 .../flink/core/memory/MemorySegmentFactory.java    |  28 +-
 .../org/apache/flink/util/CloseableIterator.java   | 109 +++-
 .../main/java/org/apache/flink/util/IOUtils.java   |  16 +
 .../java/org/apache/flink/util}/RefCounted.java    |   2 +-
 .../apache/flink/core/fs}/RefCountedFileTest.java  |  61 +-
 .../core/memory/MemorySegmentFactoryTest.java      |  64 ++
 .../apache/flink/util/CloseableIteratorTest.java   |  82 +++
 .../flink/fs/s3/common/FlinkS3FileSystem.java      |   4 +-
 .../utils/RefCountedBufferingFileStream.java       |  10 +-
 .../s3/common/utils/RefCountedFSOutputStream.java  |   1 +
 ...ntedFile.java => RefCountedFileWithStream.java} |  69 +--
 .../s3/common/utils/RefCountedTmpFileCreator.java  |  10 +-
 .../writer/S3RecoverableFsDataOutputStream.java    |  12 +-
 .../S3RecoverableMultipartUploadFactory.java       |   8 +-
 .../fs/s3/common/writer/S3RecoverableWriter.java   |   8 +-
 .../utils/RefCountedBufferingFileStreamTest.java   |   4 +-
 ...Test.java => RefCountedFileWithStreamTest.java} |  60 +-
 .../writer/RecoverableMultiPartUploadImplTest.java |   4 +-
 .../S3RecoverableFsDataOutputStreamTest.java       |  10 +-
 .../channel/ChannelStateWriteRequest.java          |  33 +-
 .../ChannelStateWriteRequestDispatcherImpl.java    |   6 +-
 .../ChannelStateWriteRequestExecutorImpl.java      |  20 +-
 .../checkpoint/channel/ChannelStateWriter.java     |   6 +-
 .../checkpoint/channel/ChannelStateWriterImpl.java |  17 +-
 .../runtime/io/disk/FileBasedBufferIterator.java   |  90 +++
 .../api/serialization/NonSpanningWrapper.java      | 372 ++++++++++++
 .../api/serialization/RecordDeserializer.java      |   4 +-
 .../network/api/serialization/SpanningWrapper.java | 314 ++++++++++
 ...SpillingAdaptiveSpanningRecordDeserializer.java | 661 ++-------------------
 .../partition/consumer/RemoteInputChannel.java     |   3 +-
 .../ChannelStateWriteRequestDispatcherTest.java    |  10 +-
 .../ChannelStateWriteRequestExecutorImplTest.java  |   1 -
 .../channel/ChannelStateWriterImplTest.java        |  13 +-
 .../channel/CheckpointInProgressRequestTest.java   |   7 +-
 .../checkpoint/channel/MockChannelStateWriter.java |  11 +-
 .../channel/RecordingChannelStateWriter.java       |  12 +-
 .../SpanningRecordSerializationTest.java           |   9 +-
 .../api/serialization/SpanningWrapperTest.java     | 115 ++++
 .../partition/consumer/SingleInputGateTest.java    |  14 +-
 .../runtime/state/ChannelPersistenceITCase.java    |   3 +-
 .../runtime/io/CheckpointBarrierUnaligner.java     |   3 +-
 .../runtime/io/StreamTaskNetworkInput.java         |  11 +-
 45 files changed, 1428 insertions(+), 937 deletions(-)
 copy {flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils => flink-core/src/main/java/org/apache/flink/core/fs}/RefCountedFile.java (60%)
 rename {flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils => flink-core/src/main/java/org/apache/flink/util}/RefCounted.java (96%)
 copy {flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils => flink-core/src/test/java/org/apache/flink/core/fs}/RefCountedFileTest.java (56%)
 create mode 100644 flink-core/src/test/java/org/apache/flink/core/memory/MemorySegmentFactoryTest.java
 create mode 100644 flink-core/src/test/java/org/apache/flink/util/CloseableIteratorTest.java
 rename flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/{RefCountedFile.java => RefCountedFileWithStream.java} (60%)
 rename flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/{RefCountedFileTest.java => RefCountedFileWithStreamTest.java} (56%)
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileBasedBufferIterator.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java
 create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java
 create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapperTest.java