[arrow] branch master updated: ARROW-12112: [CI] Reduce footprint of conda-integration image

[ks...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 0c02ff9  ARROW-12112: [CI] Reduce footprint of conda-integration image
0c02ff9 is described below

commit 0c02ff9c67d3fa981bd20b2e533280d23386347b
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Wed Apr 7 10:45:16 2021 +0200

    ARROW-12112: [CI] Reduce footprint of conda-integration image
    
    This seems to save about 2GB in the image's root directory.
    
    Closes #9891 from pitrou/ARROW-12191-conda-integration
    
    Authored-by: Antoine Pitrou <an...@python.org>
    Signed-off-by: Krisztián Szűcs <sz...@gmail.com>
---
 ci/conda_env_gandiva.yml               |  2 +-
 ci/docker/conda-integration.dockerfile | 24 +++++++++++++++++-------
 ci/scripts/util_cleanup.sh             | 19 ++++++++++++++++++-
 docker-compose.yml                     | 11 -----------
 4 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/ci/conda_env_gandiva.yml b/ci/conda_env_gandiva.yml
index 22c70a3..024b9fe 100644
--- a/ci/conda_env_gandiva.yml
+++ b/ci/conda_env_gandiva.yml
@@ -15,5 +15,5 @@
 # specific language governing permissions and limitations
 # under the License.
 
-clangdev=11
+clang=11
 llvmdev=11
diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile
index aa808da..82636e4 100644
--- a/ci/docker/conda-integration.dockerfile
+++ b/ci/docker/conda-integration.dockerfile
@@ -25,6 +25,7 @@ ARG node=14
 ARG jdk=8
 ARG go=1.15
 
+# Install Archery and integration dependencies
 COPY ci/conda_env_archery.yml /arrow/ci/
 RUN conda install -q \
         --file arrow/ci/conda_env_cpp.yml \
@@ -34,9 +35,12 @@ RUN conda install -q \
         maven=${maven} \
         nodejs=${node} \
         openjdk=${jdk} && \
-    conda clean --all
+    conda clean --all --force-pkgs-dirs
 
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+# Install Rust with only the needed components
+# (rustfmt is needed for tonic-build to compile the protobuf definitions)
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile=minimal -y && \
+    $HOME/.cargo/bin/rustup component add rustfmt
 
 ENV GOROOT=/opt/go \
     GOBIN=/opt/go/bin \
@@ -45,14 +49,20 @@ ENV GOROOT=/opt/go \
 RUN wget -nv -O - https://dl.google.com/go/go${go}.linux-${arch}.tar.gz | tar -xzf - -C /opt
 
 ENV ARROW_BUILD_INTEGRATION=ON \
+    ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \
-    ARROW_FLIGHT=ON \
-    ARROW_ORC=OFF \
+    ARROW_COMPUTE=OFF \
+    ARROW_CSV=OFF \
     ARROW_DATASET=OFF \
-    ARROW_GANDIVA=OFF \
-    ARROW_PLASMA=OFF \
     ARROW_FILESYSTEM=OFF \
+    ARROW_FLIGHT=ON \
+    ARROW_GANDIVA=OFF \
     ARROW_HDFS=OFF \
     ARROW_JEMALLOC=OFF \
     ARROW_JSON=OFF \
-    ARROW_USE_GLOG=OFF
+    ARROW_ORC=OFF \
+    ARROW_PARQUET=OFF \
+    ARROW_PLASMA=OFF \
+    ARROW_S3=OFF \
+    ARROW_USE_GLOG=OFF \
+    CMAKE_UNITY_BUILD=ON
diff --git a/ci/scripts/util_cleanup.sh b/ci/scripts/util_cleanup.sh
index 521b49e..3a13a1a 100755
--- a/ci/scripts/util_cleanup.sh
+++ b/ci/scripts/util_cleanup.sh
@@ -17,9 +17,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# this script is github actions specific to free up disk space
+# This script is Github Actions-specific to free up disk space,
+# to avoid disk full errors on some builds
 
 if [ $RUNNER_OS = "Linux" ]; then
+    df -h
+
     # remove swap
     sudo swapoff -a
     sudo rm -f /swapfile
@@ -30,8 +33,22 @@ if [ $RUNNER_OS = "Linux" ]; then
     # remove haskell, consumes 8.6 GB
     sudo rm -rf /opt/ghc
 
+    # 1 GB
+    sudo rm -rf /home/linuxbrew/.linuxbrew
+
+    # 1+ GB
+    sudo rm -rf /opt/hostedtoolcache/CodeQL
+
+    # 1+ GB
+    sudo rm -rf /usr/share/swift
+
+    # 12 GB, but takes a lot of time to delete
+    #sudo rm -rf /usr/local/lib/android
+
     # remove cached docker images, around 13 GB
     docker rmi $(docker image ls -aq)
+
+    # NOTE: /usr/share/dotnet is 25 GB
 fi
 
 df -h
diff --git a/docker-compose.yml b/docker-compose.yml
index 6ffe229..683ab88 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1193,24 +1193,13 @@ services:
     volumes: *conda-volumes
     environment:
       <<: *ccache
-      ARROW_BUILD_STATIC: "OFF"
-      ARROW_COMPUTE: "OFF"
-      ARROW_CSV: "OFF"
-      ARROW_FLIGHT: "ON"
-      ARROW_GANDIVA: "OFF"
-      ARROW_PARQUET: "OFF"
-      ARROW_S3: "OFF"
       # tell archery where the arrow binaries are located
       ARROW_CPP_EXE_PATH: /build/cpp/debug
-      CMAKE_UNITY_BUILD: "ON"
     command:
       ["/arrow/ci/scripts/rust_build.sh /arrow /build &&
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/go_build.sh /arrow &&
         /arrow/ci/scripts/java_build.sh /arrow /build &&
-        df -h &&
-        du -hx --threshold=100M / &&
-        du -hx --threshold=500M /arrow/ &&
         /arrow/ci/scripts/js_build.sh /arrow /build &&
         /arrow/ci/scripts/integration_arrow.sh /arrow /build"]