You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2020/05/17 16:49:58 UTC
[flink] branch master updated (cc8bf34 -> 547c168)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from cc8bf34 [FLINK-17649][table-planner-blink] Fix hash aggregate NPE
add 339f5d8 [FLINK-17593][fs-connector] Support arbitrary recovery mechanism for PartFileWriter
add f850ec7 [FLINK-17594][fs-connector] Support Hadoop path-based part-file writer
add 547c168 [FLINK-14255][hive] Integrate hive to streaming file sink
No new revisions were added by this update.
Summary of changes:
flink-connectors/flink-connector-hive/pom.xml | 7 +
.../hive/HiveRowDataPartitionComputer.java | 77 ++++
.../flink/connectors/hive/HiveTableFactory.java | 15 +-
.../flink/connectors/hive/HiveTableSink.java | 152 +++++---
.../hive/write/HiveBulkWriterFactory.java | 81 ++++
.../hive/write/HiveOutputFormatFactory.java | 81 ++++
.../HiveWriterFactory.java} | 165 ++++----
.../hive/HiveOutputFormatFactoryTest.java | 5 +-
.../flink/connectors/hive/HiveTableSinkTest.java | 104 +++++
.../apache/flink/core/fs/RecoverableWriter.java | 6 +-
.../core/fs/local/LocalRecoverableWriter.java | 2 +-
.../runtime/fs/hdfs/HadoopRecoverableWriter.java | 2 +-
flink-formats/flink-hadoop-bulk/pom.xml | 105 +++++
.../bulk/DefaultHadoopFileCommitterFactory.java | 37 ++
.../formats/hadoop/bulk/HadoopFileCommitter.java | 62 +++
.../hadoop/bulk/HadoopFileCommitterFactory.java | 45 +++
.../hadoop/bulk/HadoopPathBasedBulkWriter.java | 71 ++++
.../hadoop/bulk/HadoopPathBasedPartFileWriter.java | 265 +++++++++++++
.../bulk/committer/HadoopRenameFileCommitter.java | 100 +++++
.../HadoopPathBasedBulkFormatBuilder.java | 147 +++++++
.../sink/filesystem/SerializableConfiguration.java | 55 +++
.../bulk/HadoopPathBasedPartFileWriterTest.java | 191 +++++++++
.../filesystem/TestStreamingFileSinkFactory.java | 32 ++
.../src/test/resources/log4j2-test.properties | 0
flink-formats/pom.xml | 1 +
.../sink/filesystem/AbstractPartFileWriter.java | 58 +++
.../api/functions/sink/filesystem/Bucket.java | 166 ++++----
.../functions/sink/filesystem/BucketFactory.java | 7 +-
.../api/functions/sink/filesystem/BucketState.java | 32 +-
.../sink/filesystem/BucketStateSerializer.java | 146 ++++---
.../functions/sink/filesystem/BucketWriter.java | 106 +++++
.../api/functions/sink/filesystem/Buckets.java | 33 +-
.../functions/sink/filesystem/BulkPartWriter.java | 19 +-
.../sink/filesystem/DefaultBucketFactoryImpl.java | 9 +-
.../sink/filesystem/InProgressFileWriter.java | 70 ++++
.../OutputStreamBasedPartFileWriter.java | 296 ++++++++++++++
.../functions/sink/filesystem/PartFileWriter.java | 141 -------
.../sink/filesystem/RowWisePartWriter.java | 19 +-
.../sink/filesystem/StreamingFileSink.java | 21 +-
.../sink/filesystem/WriterProperties.java | 64 +++
.../sink/filesystem/BucketAssignerITCases.java | 3 +-
.../sink/filesystem/BucketStateSerializerTest.java | 429 +++++++++++++--------
.../api/functions/sink/filesystem/BucketTest.java | 58 ++-
.../api/functions/sink/filesystem/BucketsTest.java | 19 +-
.../sink/filesystem/RollingPolicyTest.java | 3 +-
.../api/functions/sink/filesystem/TestUtils.java | 57 ++-
.../filesystem/utils/NoOpRecoverableWriter.java | 2 +-
.../bucket-state-migration-test/empty-v1/snapshot | Bin 0 -> 128 bytes
.../bucket-state-migration-test/empty-v2/snapshot | Bin 0 -> 128 bytes
...inprogress.3239e0b2-b90f-4975-8c2c-283b4683b3d8 | 2 +
...inprogress.c2bf52ab-fc9e-43b1-8ba7-0c059a0a92fa | 2 +
...inprogress.822c67da-42b7-4afc-906c-3f8910c08709 | 2 +
...inprogress.4f3d9dce-c5d4-49d9-b2d3-8ccc3aa68372 | 2 +
...inprogress.2f80f3c9-281b-4a9c-b04d-a8d35e32ebce | 2 +
...inprogress.52f1eccb-6db3-4773-9585-104cab9232f9 | 1 +
.../full-v1-template/snapshot | Bin 0 -> 1613 bytes
...inprogress.5db416c2-1714-4240-8bfb-c9380ac2b5cd | 2 +
...inprogress.9414371c-1fb9-4646-8e52-83af81a1634b | 2 +
...inprogress.ae3d8565-9ecf-43e1-b70d-ddebfa20b685 | 2 +
...inprogress.347b1e40-6bde-40d4-b38f-fedcfa9322f2 | 2 +
...inprogress.cb2c7e0d-96cb-4bbc-9c4c-dd3661196c25 | 2 +
...inprogress.30e36227-42c9-4331-b919-01b7fc3a9843 | 1 +
.../full-v2-template/snapshot | Bin 0 -> 1685 bytes
...inprogress.6f11f5b0-3875-4539-870a-5a8813f5ce0f | 2 +
...inprogress.8d81b49f-f9e5-44cb-b3c7-219c56e98997 | 2 +
...inprogress.d63a2acb-950a-48ea-8f2f-4d26146f6aad | 2 +
...inprogress.23aa64c0-391d-47ee-a86b-cd04aec0aabe | 2 +
...inprogress.4851710e-50ba-42c1-b36e-b680a12d1f02 | 2 +
.../null-in-progress-v1-template/snapshot | Bin 0 -> 1504 bytes
...inprogress.7e1daa55-7749-439a-9772-cd362fadc2f9 | 2 +
...inprogress.27f35c05-b4fe-414e-b763-1fffa3d88837 | 2 +
...inprogress.8c42bb90-f7aa-484b-b550-005479a8971a | 2 +
...inprogress.ee5460e8-3994-448f-924a-682146077295 | 2 +
...inprogress.cc147575-ad14-437f-a9ab-ae36a19f67a5 | 2 +
.../null-in-progress-v2-template/snapshot | Bin 0 -> 1564 bytes
...inprogress.1b7d0fd9-64a5-46a7-9a66-8e886be61345 | 1 +
.../only-in-progress-v1/snapshot | Bin 0 -> 404 bytes
...inprogress.870be5fb-09c1-4ea7-9c92-e629b2ebfec3 | 1 +
.../only-in-progress-v2/snapshot | Bin 0 -> 416 bytes
.../table/filesystem/FileSystemTableSink.java | 8 +-
80 files changed, 2898 insertions(+), 720 deletions(-)
create mode 100644 flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveRowDataPartitionComputer.java
create mode 100644 flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/write/HiveBulkWriterFactory.java
create mode 100644 flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/write/HiveOutputFormatFactory.java
rename flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/{HiveOutputFormatFactory.java => write/HiveWriterFactory.java} (66%)
create mode 100644 flink-formats/flink-hadoop-bulk/pom.xml
create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/DefaultHadoopFileCommitterFactory.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitter.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitterFactory.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedBulkWriter.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriter.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitter.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/HadoopPathBasedBulkFormatBuilder.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/SerializableConfiguration.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java
create mode 100644 flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestStreamingFileSinkFactory.java
copy {flink-yarn => flink-formats/flink-hadoop-bulk}/src/test/resources/log4j2-test.properties (100%)
create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/AbstractPartFileWriter.java
create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketWriter.java
create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/InProgressFileWriter.java
create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/OutputStreamBasedPartFileWriter.java
delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java
create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/WriterProperties.java
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/empty-v1/snapshot
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/empty-v2/snapshot
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-0.inprogress.3239e0b2-b90f-4975-8c2c-283b4683b3d8
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-1.inprogress.c2bf52ab-fc9e-43b1-8ba7-0c059a0a92fa
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-2.inprogress.822c67da-42b7-4afc-906c-3f8910c08709
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-3.inprogress.4f3d9dce-c5d4-49d9-b2d3-8ccc3aa68372
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-4.inprogress.2f80f3c9-281b-4a9c-b04d-a8d35e32ebce
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-5.inprogress.52f1eccb-6db3-4773-9585-104cab9232f9
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/snapshot
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-0.inprogress.5db416c2-1714-4240-8bfb-c9380ac2b5cd
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-1.inprogress.9414371c-1fb9-4646-8e52-83af81a1634b
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-2.inprogress.ae3d8565-9ecf-43e1-b70d-ddebfa20b685
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-3.inprogress.347b1e40-6bde-40d4-b38f-fedcfa9322f2
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-4.inprogress.cb2c7e0d-96cb-4bbc-9c4c-dd3661196c25
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-5.inprogress.30e36227-42c9-4331-b919-01b7fc3a9843
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/snapshot
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v1-template/bucket/test-bucket/.part-0-0.inprogress.6f11f5b0-3875-4539-870a-5a8813f5ce0f
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v1-template/bucket/test-bucket/.part-0-1.inprogress.8d81b49f-f9e5-44cb-b3c7-219c56e98997
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v1-template/bucket/test-bucket/.part-0-2.inprogress.d63a2acb-950a-48ea-8f2f-4d26146f6aad
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v1-template/bucket/test-bucket/.part-0-3.inprogress.23aa64c0-391d-47ee-a86b-cd04aec0aabe
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v1-template/bucket/test-bucket/.part-0-4.inprogress.4851710e-50ba-42c1-b36e-b680a12d1f02
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v1-template/snapshot
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v2-template/bucket/test-bucket/.part-0-0.inprogress.7e1daa55-7749-439a-9772-cd362fadc2f9
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v2-template/bucket/test-bucket/.part-0-1.inprogress.27f35c05-b4fe-414e-b763-1fffa3d88837
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v2-template/bucket/test-bucket/.part-0-2.inprogress.8c42bb90-f7aa-484b-b550-005479a8971a
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v2-template/bucket/test-bucket/.part-0-3.inprogress.ee5460e8-3994-448f-924a-682146077295
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v2-template/bucket/test-bucket/.part-0-4.inprogress.cc147575-ad14-437f-a9ab-ae36a19f67a5
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/null-in-progress-v2-template/snapshot
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v1/bucket/test-bucket/.part-0-0.inprogress.1b7d0fd9-64a5-46a7-9a66-8e886be61345
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v1/snapshot
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/bucket/test-bucket/.part-0-0.inprogress.870be5fb-09c1-4ea7-9c92-e629b2ebfec3
create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/snapshot