(celeborn) branch main updated (4b60dae0f -> fde6365f6)

[re...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rexxiong pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git


    from 4b60dae0f [CELEBORN-1789][DOC] Document on Java Columnar Shuffle
     add fde6365f6 [CELEBORN-1413] Support Spark 4.0

No new revisions were added by this update.

Summary of changes:
 .github/workflows/maven.yml                        |  6 +++-
 .../spark/shuffle/celeborn/SparkCommonUtils.java   |  6 ++--
 client-spark/{spark-3 => spark-3-4}/pom.xml        |  4 +--
 .../shuffle/celeborn/CelebornShuffleDataIO.java    |  0
 .../shuffle/celeborn/HashBasedShuffleWriter.java   |  0
 .../shuffle/celeborn/SortBasedShuffleWriter.java   |  0
 .../shuffle/celeborn/SparkShuffleManager.java      |  0
 .../apache/spark/shuffle/celeborn/SparkUtils.java  |  0
 .../scala/org/apache/spark/SparkVersionUtil.scala  |  0
 .../spark/celeborn/ExceptionMakerHelper.scala      |  0
 .../CelebornShuffleFallbackPolicyRunner.scala      |  0
 .../shuffle/celeborn/CelebornShuffleHandle.scala   |  0
 .../shuffle/celeborn/CelebornShuffleReader.scala   |  0
 .../celeborn/CelebornShuffleWriterSuiteBase.java   |  0
 .../celeborn/HashBasedShuffleWriterSuiteJ.java     |  0
 .../spark/shuffle/celeborn/ShuffleManagerHook.java |  0
 .../celeborn/SortBasedShuffleWriterSuiteJ.java     |  0
 .../celeborn/TestCelebornShuffleManager.java       |  0
 .../src/test/resources/log4j.properties            |  0
 .../src/test/resources/log4j2-test.xml             |  0
 .../celeborn/CelebornShuffleManagerSuite.scala     |  0
 client-spark/spark-3-columnar-common/pom.xml       |  2 +-
 client-spark/spark-3-columnar-shuffle/pom.xml      |  2 +-
 client-spark/spark-3-shaded/pom.xml                |  2 +-
 .../pom.xml                                        |  4 +--
 .../celeborn/ColumnarHashBasedShuffleWriter.java   |  0
 .../celeborn/CelebornColumnarShuffleReader.scala   |  0
 .../columnar/CelebornColumnAccessor.scala          |  0
 .../execution/columnar/CelebornColumnBuilder.scala |  2 +-
 .../execution/columnar/CelebornColumnStats.scala   |  0
 .../execution/columnar/CelebornColumnType.scala    |  6 +++-
 .../columnar/CelebornColumnarBatchBuilder.scala    |  0
 .../CelebornColumnarBatchCodeGenBuild.scala        |  0
 .../columnar/CelebornColumnarBatchSerializer.scala |  0
 .../CelebornCompressibleColumnAccessor.scala       |  0
 .../CelebornCompressibleColumnBuilder.scala        |  0
 .../columnar/CelebornCompressionScheme.scala       |  0
 .../columnar/CelebornCompressionSchemes.scala      |  0
 .../columnar/CelebornNullableColumnAccessor.scala  |  0
 .../columnar/CelebornNullableColumnBuilder.scala   |  0
 .../ColumnarHashBasedShuffleWriterSuiteJ.java      |  0
 .../src/test/resources/log4j2-test.xml             |  0
 .../CelebornColumnarShuffleReaderSuite.scala       |  0
 .../spark-4-shaded}/pom.xml                        | 12 +++++--
 .../src/main/resources/META-INF/LICENSE            |  0
 .../src/main/resources/META-INF/NOTICE             |  0
 .../META-INF/licenses/LICENSE-protobuf.txt         |  0
 .../common/metrics/source/AbstractSource.scala     |  2 +-
 .../metrics/source/CelebornSourceSuite.scala       |  4 +--
 dev/dependencies.sh                                |  2 +-
 dev/reformat                                       |  2 ++
 .../deploy/master/http/api/v1/WorkerResource.scala |  4 +--
 pom.xml                                            | 32 +++++++++++++----
 project/CelebornBuild.scala                        | 33 +++++++++++++----
 tests/spark-it/pom.xml                             | 42 +++++++++++++++-------
 .../service/deploy/worker/PushDataHandler.scala    |  4 +--
 56 files changed, 120 insertions(+), 51 deletions(-)
 rename client-spark/{spark-3 => spark-3-4}/pom.xml (96%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/CelebornShuffleDataIO.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriter.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriter.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/scala/org/apache/spark/SparkVersionUtil.scala (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/scala/org/apache/spark/celeborn/ExceptionMakerHelper.scala (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleFallbackPolicyRunner.scala (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleHandle.scala (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/CelebornShuffleWriterSuiteBase.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/HashBasedShuffleWriterSuiteJ.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/ShuffleManagerHook.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/SortBasedShuffleWriterSuiteJ.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/test/java/org/apache/spark/shuffle/celeborn/TestCelebornShuffleManager.java (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/test/resources/log4j.properties (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/test/resources/log4j2-test.xml (100%)
 rename client-spark/{spark-3 => spark-3-4}/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleManagerSuite.scala (100%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/pom.xml (95%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/main/java/org/apache/spark/shuffle/celeborn/ColumnarHashBasedShuffleWriter.java (100%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornColumnarShuffleReader.scala (100%)
 copy client-spark/{spark-3.5-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnAccessor.scala (100%)
 copy client-spark/{spark-3.5-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnBuilder.scala (99%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnStats.scala (100%)
 copy client-spark/{spark-3.5-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnType.scala (99%)
 copy client-spark/{spark-3.5-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchBuilder.scala (100%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchCodeGenBuild.scala (100%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnarBatchSerializer.scala (100%)
 copy client-spark/{spark-3.5-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressibleColumnAccessor.scala (100%)
 copy client-spark/{spark-3.5-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressibleColumnBuilder.scala (100%)
 copy client-spark/{spark-3.5-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressionScheme.scala (100%)
 copy client-spark/{spark-3.5-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornCompressionSchemes.scala (100%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornNullableColumnAccessor.scala (100%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornNullableColumnBuilder.scala (100%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/test/java/org/apache/spark/shuffle/celeborn/ColumnarHashBasedShuffleWriterSuiteJ.java (100%)
 copy {client-flink/flink-1.14 => client-spark/spark-4-columnar-shuffle}/src/test/resources/log4j2-test.xml (100%)
 copy client-spark/{spark-3-columnar-shuffle => spark-4-columnar-shuffle}/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornColumnarShuffleReaderSuite.scala (100%)
 copy {client-flink/flink-1.18-shaded => client-spark/spark-4-shaded}/pom.xml (92%)
 copy client-spark/{spark-3-shaded => spark-4-shaded}/src/main/resources/META-INF/LICENSE (100%)
 copy client-spark/{spark-2-shaded => spark-4-shaded}/src/main/resources/META-INF/NOTICE (100%)
 copy {client-flink/flink-1.14-shaded => client-spark/spark-4-shaded}/src/main/resources/META-INF/licenses/LICENSE-protobuf.txt (100%)