(incubator-gluten) 01/01: [GLUTEN-11658][CORE] Restore scala.recompile.mode default to 'all' and introduce fast-build profile

[ch...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

changchen pushed a commit to branch GLUTEN-11658-restore-scala-recompile-mode
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git

commit f6dbcbe35785efa274c0a9f89cf17497b14535e0
Author: Chang chen <ch...@microsoft.com>
AuthorDate: Thu Feb 26 10:20:15 2026 +0000

    [GLUTEN-11658][CORE] Restore scala.recompile.mode default to 'all' and introduce fast-build profile
    
    - Restore scala.recompile.mode default from 'incremental' back to 'all'
      to fix CI/clean build failures caused by Zinc's incremental compiler
      not tracking transitive Spark dependencies correctly
    
    - Rename the 'bloop' Maven profile (added in #11645) to 'fast-build' and
      add scala.recompile.mode=incremental inside it; used by bloop-setup.sh
      and run-scala-test.sh where Zinc analysis is warm and reliable
    
    - Simplify run-scala-test.sh: replace 5 redundant -D skip flags with
      -Pfast-build profile activation
    
    - Add pathing JAR support in run-scala-test.sh to avoid OS command-line
      length limits when classpath is very long
    
    Closes #11658
    
    Co-Authored-By: Claude Sonnet 4.6 <no...@anthropic.com>
---
 dev/bloop-setup.sh                   |   2 +-
 dev/run-scala-test.sh                | 106 +++++++++++++++++++++++++----------
 docs/developers/bloop-integration.md |   2 +-
 pom.xml                              |   8 ++-
 4 files changed, 84 insertions(+), 34 deletions(-)

diff --git a/dev/bloop-setup.sh b/dev/bloop-setup.sh
index e633e2453b..8a2eb0b78c 100755
--- a/dev/bloop-setup.sh
+++ b/dev/bloop-setup.sh
@@ -190,7 +190,7 @@ fi
 # Skip style checks for faster generation
 log_step "Running generate-sources + bloopInstall..."
 ./build/mvn generate-sources bloop:bloopInstall \
-  -P"${PROFILES}",bloop \
+  -P"${PROFILES}",fast-build \
   -DskipTests
 
 if [[ ! -d ".bloop" ]]; then
diff --git a/dev/run-scala-test.sh b/dev/run-scala-test.sh
index 6f3f8688d3..cfe195f939 100755
--- a/dev/run-scala-test.sh
+++ b/dev/run-scala-test.sh
@@ -527,14 +527,9 @@ fi
 ${MVN_CMD} ${MVN_GOALS} \
   -T 1C -q \
   -pl "${MODULE}" -am \
-  -P"${PROFILES}" \
+  -P"${PROFILES},fast-build" \
   -DincludeScope=test \
   -Dmdep.outputFile="${CLASSPATH_FILE}" \
-  -Dspotless.check.skip=true \
-  -Dscalastyle.skip=true \
-  -Dcheckstyle.skip=true \
-  -Dmaven.gitcommitid.skip=true \
-  -Dremoteresources.skip=true \
   ${EXTRA_MVN_ARGS}
 
 if [[ ! -f "${CLASSPATH_FILE}" ]]; then
@@ -617,14 +612,82 @@ TIMING_STEP3=$(timer_elapsed $TIMER_STEP3_START $TIMER_STEP3_END)
 log_timing "Step 3 - Resolve classpath" "$TIMING_STEP3"
 
 # =============================================================================
-# Step 3.5: Export-only mode (if requested)
+# Step 3.5: Create pathing JAR
+# =============================================================================
+# A "pathing JAR" is a thin JAR whose MANIFEST.MF Class-Path header lists all
+# the real classpath entries as file: URIs. This avoids exceeding OS command-line
+# length limits. Works on Java 8+ (unlike @argfile which requires Java 9+).
+# Also includes META-INF/classpath.txt for human-readable review.
+# =============================================================================
+
+PATHING_JAR="/tmp/gluten-test-pathing-$$.jar"
+rm -f "${PATHING_JAR}"
+
+PATHING_MANIFEST_DIR="/tmp/gluten-test-manifest-$$"
+mkdir -p "${PATHING_MANIFEST_DIR}/META-INF"
+
+# Convert colon-separated classpath to space-separated file: URIs for manifest.
+# Also write a human-readable classpath listing (one entry per line, same order).
+CP_URIS=""
+IFS=':' read -ra _CP_ITEMS <<< "${RESOLVED_CLASSPATH}"
+: > "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt"
+
+for _item in "${_CP_ITEMS[@]}"; do
+  [[ -z "$_item" ]] && continue
+  echo "$_item" >> "${PATHING_MANIFEST_DIR}/META-INF/classpath.txt"
+  # Ensure directories end with / (required by Class-Path spec for directories)
+  [[ -d "$_item" && "$_item" != */ ]] && _item="${_item}/"
+  CP_URIS="${CP_URIS} file://${_item}"
+done
+CP_URIS="${CP_URIS# }"
+
+# Write manifest with proper 72-byte line wrapping.
+# MANIFEST.MF spec: max 72 bytes per line; continuation lines start with
+# a single space character.
+{
+  echo "Manifest-Version: 1.0"
+  _mf_header="Class-Path: ${CP_URIS}"
+  echo "${_mf_header:0:72}"
+  _mf_rest="${_mf_header:72}"
+  while [[ -n "$_mf_rest" ]]; do
+    echo " ${_mf_rest:0:71}"
+    _mf_rest="${_mf_rest:71}"
+  done
+  echo ""
+} > "${PATHING_MANIFEST_DIR}/META-INF/MANIFEST.MF"
+
+# Build the pathing JAR (manifest + human-readable classpath listing)
+(cd "${PATHING_MANIFEST_DIR}" && jar cfm "${PATHING_JAR}" META-INF/MANIFEST.MF META-INF/classpath.txt)
+rm -rf "${PATHING_MANIFEST_DIR}"
+log_info "Created pathing JAR: ${PATHING_JAR} ($(du -h "${PATHING_JAR}" | cut -f1))"
+log_info "  Review classpath: unzip -p ${PATHING_JAR} META-INF/classpath.txt"
+
+# =============================================================================
+# Build java command (shared by export-only and run modes)
+# =============================================================================
+
+JAVA_CMD="${JAVA_HOME}/bin/java"
+[[ ! -x "$JAVA_CMD" ]] && JAVA_CMD="java"
+
+JAVA_ARGS=(
+  ${JVM_ARGS}
+  "-Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties"
+  -cp "${PATHING_JAR}"
+  org.scalatest.tools.Runner
+  -oDF
+  -s "${SUITE}"
+)
+[[ -n "$TEST_METHOD" ]] && JAVA_ARGS+=(-t "${TEST_METHOD}")
+
+# =============================================================================
+# Step 3.6: Export-only mode (if requested)
 # =============================================================================
 
 if [[ "$EXPORT_ONLY" == "true" ]]; then
-  EXPORT_FILE="/tmp/gluten-classpath-exported.txt"
-  echo "$RESOLVED_CLASSPATH" > "$EXPORT_FILE"
-  log_info "✓ Classpath exported to: ${EXPORT_FILE}"
-  log_info "  Use with: java <jvm-args> -cp \"\$(cat ${EXPORT_FILE})\" org.scalatest.tools.Runner -s <suite>"
+  echo ""
+  echo -e "${YELLOW}# Run the test with:${NC}"
+  echo "${JAVA_CMD} ${JAVA_ARGS[*]}"
+  echo ""
   print_timing_summary false
   exit 0
 fi
@@ -635,12 +698,6 @@ fi
 
 log_step "Step 4: Running ScalaTest..."
 
-# Find Java
-JAVA_CMD="${JAVA_HOME}/bin/java"
-if [[ ! -x "$JAVA_CMD" ]]; then
-  JAVA_CMD="java"
-fi
-
 log_info "Suite: ${SUITE}"
 [[ -n "$TEST_METHOD" ]] && log_info "Test method: ${TEST_METHOD}"
 
@@ -650,21 +707,12 @@ echo "Running ScalaTest"
 echo "=========================================="
 echo ""
 
-# Build test method args conditionally
-TEST_METHOD_ARGS=()
-[[ -n "$TEST_METHOD" ]] && TEST_METHOD_ARGS=(-t "${TEST_METHOD}")
-
 TIMER_STEP4_START=$(timer_now)
 
 TEST_EXIT_CODE=0
-${JAVA_CMD} \
-  ${JVM_ARGS} \
-  -Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties \
-  -cp "${RESOLVED_CLASSPATH}" \
-  org.scalatest.tools.Runner \
-  -s "${SUITE}" \
-  "${TEST_METHOD_ARGS[@]}" \
-  -oDF || TEST_EXIT_CODE=$?
+${JAVA_CMD} "${JAVA_ARGS[@]}" || TEST_EXIT_CODE=$?
+
+rm -f "${PATHING_JAR}"
 
 TIMER_STEP4_END=$(timer_now)
 TIMING_STEP4=$(timer_elapsed $TIMER_STEP4_START $TIMER_STEP4_END)
diff --git a/docs/developers/bloop-integration.md b/docs/developers/bloop-integration.md
index 2364b8f596..c5efdbbd52 100644
--- a/docs/developers/bloop-integration.md
+++ b/docs/developers/bloop-integration.md
@@ -65,7 +65,7 @@ The `-Pbloop` profile automatically skips style checks during configuration gene
 ./dev/bloop-setup.sh -Pspark-3.5,scala-2.12,backends-velox
 
 # Manual invocation with profile
-./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,bloop -DskipTests
+./build/mvn generate-sources bloop:bloopInstall -Pspark-3.5,scala-2.12,backends-velox,fast-build -DskipTests
 ```
 
 The bloop profile sets these properties automatically:
diff --git a/pom.xml b/pom.xml
index 9af9927f9e..0f48bb5827 100644
--- a/pom.xml
+++ b/pom.xml
@@ -97,7 +97,7 @@
     -->
     <os.full.name>unknown</os.full.name>
     <!-- To build built-in backend c++ codes -->
-    <scala.recompile.mode>incremental</scala.recompile.mode>
+    <scala.recompile.mode>all</scala.recompile.mode>
     <!-- For unit tests -->
     <fasterxml.version>2.15.0</fasterxml.version>
     <junit.version>4.13.1</junit.version>
@@ -2147,17 +2147,19 @@
       </build>
     </profile>
     <profile>
-      <id>bloop</id>
+      <id>fast-build</id>
       <activation>
         <activeByDefault>false</activeByDefault>
       </activation>
       <properties>
-        <!-- Skip style checks during bloop configuration generation -->
+        <!-- Skip style checks for faster local builds -->
         <spotless.check.skip>true</spotless.check.skip>
         <scalastyle.skip>true</scalastyle.skip>
         <checkstyle.skip>true</checkstyle.skip>
         <maven.gitcommitid.skip>true</maven.gitcommitid.skip>
         <remoteresources.skip>true</remoteresources.skip>
+        <!-- Use incremental Scala compilation (Zinc state is warm in local/dev contexts) -->
+        <scala.recompile.mode>incremental</scala.recompile.mode>
       </properties>
     </profile>
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org