You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by fe...@apache.org on 2025/06/09 19:21:48 UTC
(celeborn) tag v0.6.0-rc0 updated (1245b88a6 -> 96943300f)
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a change to tag v0.6.0-rc0
in repository https://gitbox.apache.org/repos/asf/celeborn.git
*** WARNING: tag v0.6.0-rc0 was modified! ***
from 1245b88a6 (commit)
to 96943300f (commit)
from 1245b88a6 [RELEASE] Bump 0.6.0
add 1db8e6164 [RELEASE] Bump 0.6.1-SNAPSHOT
add 84c2c9f6a [CELEBORN-2013] Upgrade scala binary version of spark-3.3, spark-3.4, spark-3.5 profile to 2.13.8
add f7baeb11b [MINOR][DOC] Correct configuration values in slotsallocation
add 8c219c549 [CELEBORN-2015] Retry IOException failures for RPC requests
add c0a8cf02b [CELEBORN-1528][HELM] Use volume claim template to support various storage backend
add a3fc15939 [CELEBORN-2009] Commit files request failure should exclude worker in LifecycleManager
add 161cd9fab [CELEBORN-1413][FOLLOWUP] Bump spark 4.0 version to 4.0.0
add 3b7522a23 [CELEBORN-1413][FOLLOWUP] Support dependencies of spark-4.0 profile
add de7949013 [CELEBORN-1627][FOLLOWUP] Fix the issue where the case of name affects the metrics dashboard
add a209d9eea [CELEBORN-2020] Support http authentication for Celeborn CLI
add 4d989f425 [CELEBORN-2005] Introduce numBytesIn, numBytesOut, numBytesInPerSecond, numBytesOutPerSecond metrics for RemoteShuffleServiceFactory
add 3db9bf31b [CELEBORN-2005][FOLLOWUP] Introduce ShuffleMetricGroup for numBytesIn, numBytesOut, numRecordsOut, numBytesInPerSecond, numBytesOutPerSecond, numRecordsOutPerSecond metrics
add 1a14d1fb8 [CELEBORN-2018] Support min number of workers selected for shuffle
add 1dbf31c96 [CELEBORN-2023] Spark4 Client incompatible with isLocalMaster method
add e1f4a75de [CELEBORN-2020][FOLLOWUP] Fix CLI master commands authentication testing
add 870da8655 [CELEBORN-2021] Fix issues on regression HDFS and OSS before release 0.6
add 215b51ae1 [CELEBORN-2022] Spark4 Client should package commons-io
add 041b5c359 [CELEBORN-2020][FOLLOWUP] Add --auth-header option to usage of CLI commands
add ce2cd1869 [CELEBORN-2026] Skip build tez client
add ba84b5be6 [CELEBORN-2025] RpcFailure Scala 2.13 serialization is incompatible
add b55d63cb7 [CELEBORN-2017][HELM] Add namespace to the metadata
add d3479e1b2 [CELEBORN-2003] Add retry mechanism when completing S3 multipart upload
add ac7cee376 [CELEBORN-1673][FOLLOWUP] Shouldn't ignore InterruptedException when client retry
add daed456ae [CELEBORN-1720][FOLLOWUP] Fix flakyTest - check if fetch failure task another attempt is running or successful
add 20966c78d [CELEBORN-2027] Allow CelebornShuffleReader to decompress data on demand
add d10080238 [CELEBORN-2029][FLINK] Some minor optimizations in the Flink integration
add 7015fa086 [CELEBORN-2030] Bump Spark from 3.5.5 to 3.5.6
add 705fe4978 [CELEBORN-2015][FOLLOWUP] Retry IOException failures for RPC requests
add 96943300f [RELEASE] Bump 0.6.0
No new revisions were added by this update.
Summary of changes:
.github/workflows/deps.yml | 20 +++
.github/workflows/maven.yml | 37 +++++
.github/workflows/sbt.yml | 42 +++++-
LICENSE-binary | 4 +
README.md | 15 +-
build/make-distribution.sh | 8 +-
build/release/release.sh | 8 +-
charts/celeborn/Chart.yaml | 2 +-
charts/celeborn/ci/values.yaml | 26 ----
charts/celeborn/templates/_helpers.tpl | 32 ++---
charts/celeborn/templates/configmap.yaml | 3 +-
charts/celeborn/templates/master/podmonitor.yaml | 1 +
.../celeborn/templates/master/priorityclass.yaml | 1 +
charts/celeborn/templates/master/service.yaml | 2 +
charts/celeborn/templates/master/statefulset.yaml | 35 +++--
charts/celeborn/templates/role.yaml | 8 +-
charts/celeborn/templates/rolebinding.yaml | 9 +-
charts/celeborn/templates/serviceaccount.yaml | 3 +-
charts/celeborn/templates/worker/podmonitor.yaml | 1 +
.../celeborn/templates/worker/priorityclass.yaml | 1 +
charts/celeborn/templates/worker/service.yaml | 1 +
charts/celeborn/templates/worker/statefulset.yaml | 34 +++--
charts/celeborn/tests/configmap_test.yaml | 1 +
charts/celeborn/tests/master/podmonitor_test.yaml | 2 +
.../celeborn/tests/master/priorityclass_test.yaml | 2 +
charts/celeborn/tests/master/service_test.yaml | 2 +
charts/celeborn/tests/master/statefulset_test.yaml | 61 ++++++++
charts/celeborn/tests/role_test.yaml | 21 +--
charts/celeborn/tests/rolebinding_test.yaml | 16 +--
charts/celeborn/tests/serviceaccount_test.yaml | 16 ++-
charts/celeborn/tests/worker/podmonitor_test.yaml | 1 +
.../celeborn/tests/worker/priorityclass_test.yaml | 2 +
charts/celeborn/tests/worker/service_test.yaml | 2 +
charts/celeborn/tests/worker/statefulset_test.yaml | 61 ++++++++
charts/celeborn/values.yaml | 158 ++++++++++++++-------
cli/pom.xml | 7 +
.../apache/celeborn/cli/common/CommonOptions.scala | 17 +++
.../celeborn/cli/master/MasterSubcommandImpl.scala | 44 +++---
.../celeborn/cli/worker/WorkerSubcommandImpl.scala | 26 ++--
.../celeborn/cli/TestCelebornCliCommands.scala | 35 ++++-
.../flink/AbstractRemoteShuffleInputGate.java | 13 +-
.../AbstractRemoteShuffleInputGateFactory.java | 18 ++-
...bstractRemoteShuffleResultPartitionFactory.java | 21 +--
.../plugin/flink/RemoteShuffleEnvironment.java | 35 ++++-
.../flink/RemoteShuffleInputGateDelegation.java | 30 +++-
.../plugin/flink/RemoteShuffleOutputGate.java | 45 ++++--
.../RemoteShuffleResultPartitionDelegation.java | 8 ++
.../flink/client/FlinkShuffleClientImpl.java | 10 +-
.../metrics/dump/ShuffleQueryScopeInfo.java | 64 +++++++++
.../metrics/groups/ShuffleIOMetricGroup.java | 92 ++++++++++++
.../runtime/metrics/groups/ShuffleMetricGroup.java | 114 +++++++++++++++
.../runtime/metrics/scope/ShuffleScopeFormat.java | 78 ++++++++++
.../flink/RemoteShuffleOutputGateSuiteJ.java | 57 +++++++-
.../plugin/flink/RemoteShuffleInputGate.java | 13 +-
.../flink/RemoteShuffleInputGateFactory.java | 15 +-
.../flink/RemoteShuffleResultPartitionFactory.java | 7 +-
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 58 +++++---
.../plugin/flink/RemoteShuffleInputGate.java | 13 +-
.../flink/RemoteShuffleInputGateFactory.java | 15 +-
.../flink/RemoteShuffleResultPartitionFactory.java | 7 +-
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 22 ++-
.../plugin/flink/RemoteShuffleInputGate.java | 13 +-
.../flink/RemoteShuffleInputGateFactory.java | 15 +-
.../flink/RemoteShuffleResultPartitionFactory.java | 7 +-
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 22 ++-
.../plugin/flink/RemoteShuffleInputGate.java | 13 +-
.../flink/RemoteShuffleInputGateFactory.java | 15 +-
.../flink/RemoteShuffleResultPartitionFactory.java | 7 +-
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 22 ++-
.../plugin/flink/RemoteShuffleInputGate.java | 13 +-
.../flink/RemoteShuffleInputGateFactory.java | 15 +-
.../flink/RemoteShuffleResultPartitionFactory.java | 7 +-
.../flink/tiered/CelebornTierProducerAgent.java | 3 +
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 22 ++-
.../plugin/flink/RemoteShuffleInputGate.java | 13 +-
.../flink/RemoteShuffleInputGateFactory.java | 15 +-
.../flink/RemoteShuffleResultPartitionFactory.java | 7 +-
.../flink/tiered/CelebornTierProducerAgent.java | 3 +
.../flink/RemoteShuffleResultPartitionSuiteJ.java | 22 ++-
client-mr/mr/pom.xml | 4 +
.../apache/spark/shuffle/celeborn/SparkUtils.java | 4 +
client-spark/spark-3-columnar-shuffle/pom.xml | 10 ++
client-spark/spark-3/pom.xml | 10 ++
.../shuffle/celeborn/SparkShuffleManager.java | 3 +-
.../apache/spark/shuffle/celeborn/SparkUtils.java | 11 ++
.../shuffle/celeborn/CelebornShuffleReader.scala | 27 +++-
.../execution/columnar/CelebornColumnType.scala | 2 +-
client-spark/spark-4-shaded/pom.xml | 1 +
client-tez/tez/pom.xml | 4 +
.../apache/celeborn/client/DummyShuffleClient.java | 3 +-
.../org/apache/celeborn/client/ShuffleClient.java | 6 +-
.../apache/celeborn/client/ShuffleClientImpl.java | 6 +-
.../celeborn/client/read/CelebornInputStream.java | 30 ++--
.../celeborn/client/read/DfsPartitionReader.java | 12 +-
.../celeborn/client/ApplicationHeartbeater.scala | 2 +-
.../apache/celeborn/client/LifecycleManager.scala | 15 +-
.../celeborn/client/WorkerStatusTracker.scala | 2 +
.../celeborn/client/commit/CommitHandler.scala | 9 ++
.../celeborn/client/WithShuffleClientSuite.scala | 6 +-
.../network/client/TransportClientFactory.java | 11 +-
.../common/protocol/message/StatusCode.java | 3 +-
.../org/apache/celeborn/common/CelebornConf.scala | 42 +++++-
.../celeborn/common/metrics/source/Role.scala | 6 +-
.../celeborn/common/rpc/RpcEndpointRef.scala | 34 +----
.../org/apache/celeborn/common/rpc/RpcEnv.scala | 33 +----
.../celeborn/common/rpc/netty/NettyRpcEnv.scala | 1 +
.../org/apache/celeborn/common/util/Utils.scala | 35 +++++
dev/dependencies.sh | 4 +
dev/deps/dependencies-client-mr | 2 +-
...ent-flink-2.0 => dependencies-client-spark-4.0} | 8 +-
dev/deps/dependencies-client-tez | 2 +-
dev/deps/dependencies-server | 3 +-
docs/celeborn_cli.md | 36 +++--
docs/configuration/client.md | 1 +
docs/configuration/master.md | 3 +-
docs/configuration/worker.md | 2 +
docs/developers/sbt.md | 6 +
docs/developers/slotsallocation.md | 4 +-
docs/migration.md | 4 +-
.../celeborn/service/deploy/master/Master.scala | 16 ++-
.../deploy/master/quota/QuotaManagerSuite.scala | 22 +--
.../apache/celeborn/S3MultipartUploadHandler.java | 59 +++++++-
pom.xml | 33 ++++-
project/CelebornBuild.scala | 68 ++++++---
service/pom.xml | 24 ++--
tests/spark-it/pom.xml | 94 +++++++++++-
.../celeborn/tests/spark/SparkTestBase.scala | 9 +-
.../spark/shuffle/celeborn/SparkUtilsSuite.scala | 6 +-
.../deploy/worker/storage/MapPartitionData.java | 17 ++-
.../worker/storage/MapPartitionDataReader.java | 13 +-
.../deploy/worker/storage/TierWriterHelper.java | 11 +-
.../deploy/worker/storage/StoragePolicy.scala | 2 +-
.../service/deploy/worker/storage/TierWriter.scala | 9 +-
.../storage/PartitionDataWriterSuiteUtils.java | 4 +-
.../cluster/LocalReadByChunkOffsetsTest.scala | 3 +-
.../service/deploy/cluster/ReadWriteTestBase.scala | 3 +-
.../deploy/cluster/ReadWriteTestWithFailures.scala | 3 +-
137 files changed, 1992 insertions(+), 547 deletions(-)
create mode 100644 client-flink/common/src/main/java/org/apache/flink/runtime/metrics/dump/ShuffleQueryScopeInfo.java
create mode 100644 client-flink/common/src/main/java/org/apache/flink/runtime/metrics/groups/ShuffleIOMetricGroup.java
create mode 100644 client-flink/common/src/main/java/org/apache/flink/runtime/metrics/groups/ShuffleMetricGroup.java
create mode 100644 client-flink/common/src/main/java/org/apache/flink/runtime/metrics/scope/ShuffleScopeFormat.java
copy dev/deps/{dependencies-client-flink-2.0 => dependencies-client-spark-4.0} (95%)