You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/02/01 23:14:32 UTC

(spark-kubernetes-operator) branch main updated: [SPARK-55298] Fix "Argument list too long" error in `assertGeneratedCRDMatchesHelmChart` task

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git


The following commit(s) were added to refs/heads/main by this push:
     new 3cf1ec3  [SPARK-55298] Fix "Argument list too long" error in `assertGeneratedCRDMatchesHelmChart` task
3cf1ec3 is described below

commit 3cf1ec3fa09c1934efaf0e15b5a468348aa95faf
Author: Zhou JIANG <ji...@umich.edu>
AuthorDate: Sun Feb 1 15:14:26 2026 -0800

    [SPARK-55298] Fix "Argument list too long" error in `assertGeneratedCRDMatchesHelmChart` task
    
    ### What changes were proposed in this pull request?
    
    This PR fixes the "Argument list too long" error in the `assertGeneratedCRDMatchesHelmChart` Gradle task that occurs occasionally when comparing generated CRDs with staged versions.
    
    ### Why are the changes needed?
    
    The previous implementation embedded entire CRD YAML content (potentially hundreds of KB) directly into bash command arguments using `echo '${generatedAppCRD}'`. This may exceeded the OS argument length limit in some environments, causing the build to fail with:
    
    ```
    java.io.IOException: Cannot run program "bash": Exec failed, error: 7 (Argument list too long)
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This is an internal build task fix that doesn't affect runtime behavior or user-facing features.
    
    ### How was this patch tested?
    
    The fix was tested by:
    
    1. Verifying the task still correctly detects CRD differences when the diff is large
    2. Confirming temp files are properly created and cleaned up
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No
    
    Closes #479 from jiangzho/prune_error_messages.
    
    Authored-by: Zhou JIANG <ji...@umich.edu>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 spark-operator-api/build.gradle | 40 ++++++++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/spark-operator-api/build.gradle b/spark-operator-api/build.gradle
index 03596fe..b313fc5 100644
--- a/spark-operator-api/build.gradle
+++ b/spark-operator-api/build.gradle
@@ -94,23 +94,39 @@ tasks.register("assertGeneratedCRDMatchesHelmChart") {
 
       if (generatedAppCRD != stagedAppCRD) {
         errorMessage.append("=== SparkApplication CRD Differences ===\n")
-        def appDiff = ["bash", "-c",
-                       "diff -u <(echo '${generatedAppCRD.replace("'", "'\\''")}' " +
-                           "| yq -P 'sort_keys(..)') <(echo '${stagedAppCRD.replace("'", "'\\''")}' " +
-                           "| yq -P 'sort_keys(..)')"]
-            .execute().text
-        errorMessage.append(appDiff ?: "Unable to generate diff\n")
+        def tmpGenerated = File.createTempFile("generated-app-", ".yaml")
+        def tmpStaged = File.createTempFile("staged-app-", ".yaml")
+        try {
+          tmpGenerated.text = generatedAppCRD
+          tmpStaged.text = stagedAppCRD
+          def appDiff = ["bash", "-c",
+                         "diff -u <(yq -P 'sort_keys(..)' ${tmpGenerated.absolutePath}) " +
+                             "<(yq -P 'sort_keys(..)' ${tmpStaged.absolutePath})"]
+              .execute().text
+          errorMessage.append(appDiff ?: "Unable to generate diff\n")
+        } finally {
+          tmpGenerated.delete()
+          tmpStaged.delete()
+        }
         errorMessage.append("\n")
       }
 
       if (generatedClusterCRD != stagedClusterCRD) {
         errorMessage.append("=== SparkCluster CRD Differences ===\n")
-        def clusterDiff = ["bash", "-c",
-                           "diff -u <(echo '${generatedClusterCRD.replace("'", "'\\''")}' " +
-                               "| yq -P 'sort_keys(..)') <(echo '${stagedClusterCRD.replace("'", "'\\''")}' " +
-                               "| yq -P 'sort_keys(..)')"]
-            .execute().text
-        errorMessage.append(clusterDiff ?: "Unable to generate diff\n")
+        def tmpGenerated = File.createTempFile("generated-cluster-", ".yaml")
+        def tmpStaged = File.createTempFile("staged-cluster-", ".yaml")
+        try {
+          tmpGenerated.text = generatedClusterCRD
+          tmpStaged.text = stagedClusterCRD
+          def clusterDiff = ["bash", "-c",
+                             "diff -u <(yq -P 'sort_keys(..)' ${tmpGenerated.absolutePath}) " +
+                                 "<(yq -P 'sort_keys(..)' ${tmpStaged.absolutePath})"]
+              .execute().text
+          errorMessage.append(clusterDiff ?: "Unable to generate diff\n")
+        } finally {
+          tmpGenerated.delete()
+          tmpStaged.delete()
+        }
       }
 
       throw new GradleException(errorMessage.toString())


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org