You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by vi...@apache.org on 2024/02/22 20:55:10 UTC
(arrow-datafusion-comet) branch main updated: build: Upgrade spotless version to 2.43.0 (#85)
This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 074b5fd build: Upgrade spotless version to 2.43.0 (#85)
074b5fd is described below
commit 074b5fd4cf7abd5bb0d772f21564837032764de4
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Thu Feb 22 12:55:04 2024 -0800
build: Upgrade spotless version to 2.43.0 (#85)
Current spotless `2.29.0` doesn't work with Apache Maven 3.9.6 + Java version: 21.0.2.
```
java.lang.reflect.InvocationTargetException
at jdk.internal.reflect.DirectMethodHandleAccessor.invoke (DirectMethodHandleAccessor.java:118)
at java.lang.reflect.Method.invoke (Method.java:580)
...
Caused by: java.lang.NoSuchMethodError: 'com.sun.tools.javac.tree.JCTree com.sun.tools.javac.tree.JCTree$JCImport.getQualifiedIdentifier()'
...
```
---
.github/actions/java-test/action.yaml | 12 ++++++++++--
.github/actions/rust-test/action.yaml | 6 +++++-
pom.xml | 2 +-
.../spark/shuffle/comet/CometShuffleMemoryAllocator.java | 1 +
.../comet/execution/shuffle/CometUnsafeShuffleWriter.java | 1 +
5 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
index 9c7de9a..ae6d80d 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -38,9 +38,17 @@ runs:
shell: bash
run: |
echo "JAVA_VERSION=${JAVA_VERSION}"
- ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION}
+ if [ $JAVA_VERSION == "1.8" ]; then
+ ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0
+ else
+ ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION}
+ fi
- name: Run tests
shell: bash
run: |
- SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION}
+ if [ $JAVA_VERSION == "1.8" ]; then
+ SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0
+ else
+ SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION}
+ fi
diff --git a/.github/actions/rust-test/action.yaml b/.github/actions/rust-test/action.yaml
index a013a7b..cb6c29c 100644
--- a/.github/actions/rust-test/action.yaml
+++ b/.github/actions/rust-test/action.yaml
@@ -51,7 +51,11 @@ runs:
shell: bash
run: |
cd common
- ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION}
+ if [ $JAVA_VERSION == "1.8" ]; then
+ ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0
+ else
+ ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION}
+ fi
- name: Run Cargo test
shell: bash
diff --git a/pom.xml b/pom.xml
index b3c3545..507ec78 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@ under the License.
<parquet.maven.scope>provided</parquet.maven.scope>
<arrow.version>14.0.2</arrow.version>
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
- <spotless.version>2.29.0</spotless.version>
+ <spotless.version>2.43.0</spotless.version>
<jni.dir>${project.basedir}/../core/target/debug</jni.dir>
<platform>darwin</platform>
<arch>x86_64</arch>
diff --git a/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocator.java b/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocator.java
index e8fe170..2837fa3 100644
--- a/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocator.java
+++ b/spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocator.java
@@ -52,6 +52,7 @@ public final class CometShuffleMemoryAllocator extends MemoryConsumer {
/** The number of bits used to address the page table. */
private static final int PAGE_NUMBER_BITS = 13;
+
/** The number of entries in the page table. */
private static final int PAGE_TABLE_SIZE = 1 << PAGE_NUMBER_BITS;
diff --git a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java
index ae38e4f..7690e1d 100644
--- a/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java
+++ b/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java
@@ -121,6 +121,7 @@ public class CometUnsafeShuffleWriter<K, V> extends ShuffleWriter<K, V> {
private long peakMemoryUsedBytes = 0;
private ExposedByteArrayOutputStream serBuffer;
private SerializationStream serOutputStream;
+
/**
* Are we in the process of stopping? Because map tasks can call stop() with success = true and
* then call stop() with success = false if they get an exception, we want to make sure we don't