You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2022/02/15 16:54:52 UTC
[GitHub] [arrow] pitrou commented on a change in pull request #12320: ARROW-15483: [Release] Revamp the verification scripts
pitrou commented on a change in pull request #12320:
URL: https://github.com/apache/arrow/pull/12320#discussion_r807034394
##########
File path: dev/release/setup-rhel-rebuilds.sh
##########
@@ -46,10 +46,10 @@ dnf -y install \
ninja-build \
nodejs \
openssl-devel \
- python3-devel \
- python3-devel \
- python3-pip \
- python3-pip \
+ python38-devel \
+ python38-devel \
+ python38-pip \
+ python38-pip \
Review comment:
Is there a particular reason to force Python 3.8 here?
##########
File path: dev/release/verify-release-candidate.bat
##########
@@ -20,30 +20,49 @@
@echo on
+setlocal ENABLEDELAYEDEXPANSION
+
if not exist "C:\tmp\" mkdir C:\tmp
if exist "C:\tmp\arrow-verify-release" rd C:\tmp\arrow-verify-release /s /q
if not exist "C:\tmp\arrow-verify-release" mkdir C:\tmp\arrow-verify-release
set _VERIFICATION_DIR=C:\tmp\arrow-verify-release
set _VERIFICATION_DIR_UNIX=C:/tmp/arrow-verify-release
set _VERIFICATION_CONDA_ENV=%_VERIFICATION_DIR%\conda-env
-set _DIST_URL=https://dist.apache.org/repos/dist/dev/arrow
-set _TARBALL=apache-arrow-%1.tar.gz
-set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1
set INSTALL_DIR=%_VERIFICATION_DIR%\install
-@rem Requires GNU Wget for Windows
-wget --no-check-certificate -O %_TARBALL% %_DIST_URL%/apache-arrow-%1-rc%2/%_TARBALL% || exit /B 1
-
-tar xf %_TARBALL% -C %_VERIFICATION_DIR_UNIX%
-
+set VERSION=%1
+set RC_NUMBER=%2
+
+if "%VERSION%"=="" (
+ set ARROW_SOURCE=%~dp0..\..\
+) else (
+ set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1
+ if "%RC_NUMBER%"=="" (
+ @rem verify a specific git revision
+ git clone --recurse-submodules https://github.com/apache/arrow.git !ARROW_SOURCE!
+ git -C !ARROW_SOURCE! checkout %VERSION%
+ ) else (
+ @rem verify a release candidate tarball
+ @rem Requires GNU Wget for Windows
+ set TARBALL_NAME=apache-arrow-%VERSION%.tar.gz
+ set TARBALL_URL=https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-%VERSION%-rc%RC_NUMBER%/%TARBALL_NAME%
+ wget --no-check-certificate -O %TARBALL_NAME% %TARBALL_URL% || exit /B 1
+ tar xf %TARBALL_NAME% -C %_VERIFICATION_DIR_UNIX%
+ )
+ git clone https://github.com/apache/arrow-testing.git !ARROW_SOURCE!\testing
+ git clone https://github.com/apache/parquet-testing.git !ARROW_SOURCE!\cpp\submodules\parquet-testing
Review comment:
This is only useful for the tarball case, since you used `--recurse-submodules` above, isn't it?
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
Review comment:
```suggestion
$python -m venv ${virtualenv}
```
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_CUDA=${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-$DEFAULT_DEPENDENCY_SOURCE} \
+ -DARROW_FLIGHT=${ARROW_FLIGHT} \
+ -DARROW_GANDIVA=${ARROW_GANDIVA} \
+ -DARROW_HDFS=ON \
+ -DARROW_ORC=ON \
+ -DARROW_PARQUET=ON \
+ -DARROW_PLASMA=${ARROW_PLASMA} \
+ -DARROW_PYTHON=ON \
+ -DARROW_S3=${ARROW_S3} \
+ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-OFF} \
Review comment:
Why off by default?
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_CUDA=${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-$DEFAULT_DEPENDENCY_SOURCE} \
+ -DARROW_FLIGHT=${ARROW_FLIGHT} \
+ -DARROW_GANDIVA=${ARROW_GANDIVA} \
+ -DARROW_HDFS=ON \
+ -DARROW_ORC=ON \
+ -DARROW_PARQUET=ON \
+ -DARROW_PLASMA=${ARROW_PLASMA} \
+ -DARROW_PYTHON=ON \
+ -DARROW_S3=${ARROW_S3} \
+ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-OFF} \
+ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
+ -DARROW_WITH_BROTLI=ON \
+ -DARROW_WITH_BZ2=ON \
+ -DARROW_WITH_LZ4=ON \
+ -DARROW_WITH_SNAPPY=ON \
+ -DARROW_WITH_ZLIB=ON \
+ -DARROW_WITH_ZSTD=ON \
+ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-release} \
+ -DCMAKE_INSTALL_LIBDIR=lib \
+ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
+ -DGTest_SOURCE=BUNDLED \
+ -DPARQUET_REQUIRE_ENCRYPTION=ON \
+ ${ARROW_CMAKE_OPTIONS:-} \
+ ${ARROW_SOURCE_DIR}/cpp
+ cmake --build . --target install
+
+ # TODO: ARROW-5036: plasma-serialization_tests broken
+ # TODO: ARROW-5054: libgtest.so link failure in flight-server-test
+ LD_LIBRARY_PATH=$PWD/release:$LD_LIBRARY_PATH ctest \
+ --exclude-regex "plasma-serialization_tests" \
+ -j$NPROC \
+ --output-on-failure \
+ -L unittest
+
+ popd
+}
test_python() {
- pushd python
+ # Build and test Python
+ setup_virtualenv cython numpy setuptools_scm setuptools
+ setup_conda --file ci/conda_env_python.txt
export PYARROW_PARALLEL=$NPROC
-
export PYARROW_WITH_DATASET=1
+ export PYARROW_WITH_HDFS=1
+ export PYARROW_WITH_ORC=1
export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_PLASMA=1
+ if [ "${ARROW_S3}" = "ON" ]; then
Review comment:
Can you keep these entries alphabetically ordered?
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -66,14 +96,18 @@ detect_cuda() {
return $((${n_gpus} < 1))
}
-# Build options for the C++ library
+# Execute tests in a conda enviroment
+: ${USE_CONDA:=0}
+# Build options for the C++ library
if [ -z "${ARROW_CUDA:-}" ] && detect_cuda; then
ARROW_CUDA=ON
fi
+: ${ARROW_S3:=OFF}
Review comment:
Keep these lines alphabetically ordered?
##########
File path: dev/tasks/verify-rc/github.macos.amd64.yml
##########
@@ -31,20 +33,36 @@ jobs:
{% endif %}
steps:
- {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
+ {% if not use_conda %}
- name: Install System Dependencies
shell: bash
run: |
brew update
brew bundle --file=arrow/cpp/Brewfile
brew bundle --file=arrow/c_glib/Brewfile
+ {% endif %}
+
+ - uses: actions/setup-java@v2
+ with:
+ distribution: 'temurin'
+ java-version: '11'
+
- uses: actions/setup-node@v2-beta
with:
node-version: '16'
+
- name: Run verification
shell: bash
+ env:
+ TEST_DEFAULT: 0
+ {% if target is defined %}
Review comment:
If it's not defined then nothing is tested?
##########
File path: dev/tasks/verify-rc/github.macos.amd64.yml
##########
@@ -31,20 +33,36 @@ jobs:
{% endif %}
steps:
- {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
+ {% if not use_conda %}
- name: Install System Dependencies
shell: bash
run: |
brew update
brew bundle --file=arrow/cpp/Brewfile
brew bundle --file=arrow/c_glib/Brewfile
+ {% endif %}
+
+ - uses: actions/setup-java@v2
Review comment:
Is this unconditional even if Java is not being tested? (same for other setup steps)
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
Review comment:
Does this automatically invoke the `mamba()` function above?
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_CUDA=${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-$DEFAULT_DEPENDENCY_SOURCE} \
+ -DARROW_FLIGHT=${ARROW_FLIGHT} \
+ -DARROW_GANDIVA=${ARROW_GANDIVA} \
+ -DARROW_HDFS=ON \
Review comment:
```suggestion
-DARROW_GCS=${ARROW_GCS} \
-DARROW_HDFS=ON \
```
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_CUDA=${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-$DEFAULT_DEPENDENCY_SOURCE} \
+ -DARROW_FLIGHT=${ARROW_FLIGHT} \
+ -DARROW_GANDIVA=${ARROW_GANDIVA} \
+ -DARROW_HDFS=ON \
+ -DARROW_ORC=ON \
+ -DARROW_PARQUET=ON \
+ -DARROW_PLASMA=${ARROW_PLASMA} \
+ -DARROW_PYTHON=ON \
+ -DARROW_S3=${ARROW_S3} \
+ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-OFF} \
+ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
+ -DARROW_WITH_BROTLI=ON \
+ -DARROW_WITH_BZ2=ON \
+ -DARROW_WITH_LZ4=ON \
Review comment:
```suggestion
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_RE2=ON \
```
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_CUDA=${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-$DEFAULT_DEPENDENCY_SOURCE} \
+ -DARROW_FLIGHT=${ARROW_FLIGHT} \
+ -DARROW_GANDIVA=${ARROW_GANDIVA} \
+ -DARROW_HDFS=ON \
+ -DARROW_ORC=ON \
+ -DARROW_PARQUET=ON \
+ -DARROW_PLASMA=${ARROW_PLASMA} \
+ -DARROW_PYTHON=ON \
+ -DARROW_S3=${ARROW_S3} \
+ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-OFF} \
+ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
+ -DARROW_WITH_BROTLI=ON \
+ -DARROW_WITH_BZ2=ON \
+ -DARROW_WITH_LZ4=ON \
+ -DARROW_WITH_SNAPPY=ON \
Review comment:
```suggestion
-DARROW_WITH_SNAPPY=ON \
-DARROW_WITH_UTF8PROC=ON \
```
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_CUDA=${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-$DEFAULT_DEPENDENCY_SOURCE} \
+ -DARROW_FLIGHT=${ARROW_FLIGHT} \
+ -DARROW_GANDIVA=${ARROW_GANDIVA} \
+ -DARROW_HDFS=ON \
+ -DARROW_ORC=ON \
Review comment:
```suggestion
-DARROW_JSON=ON \
-DARROW_ORC=ON \
```
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -404,58 +599,75 @@ test_python() {
export PYARROW_WITH_GANDIVA=1
fi
+ pushd python
+
+ # Build pyarrow
python setup.py build_ext --inplace
- pytest pyarrow -v --pdb
+
+ # Check mandatory and optional imports
+ python -c "
+import pyarrow
+import pyarrow._hdfs
+import pyarrow.csv
+import pyarrow.dataset
+import pyarrow.fs
+import pyarrow.json
+import pyarrow.orc
+import pyarrow.parquet
+import pyarrow.plasma
+"
+ if [ "${PYARROW_WITH_S3}" == "ON" ]; then
+ python -c "import pyarrow._s3fs"
+ fi
+ if [ "${PYARROW_WITH_CUDA}" == "ON" ]; then
+ python -c "import pyarrow.cuda"
+ fi
+ if [ "${PYARROW_WITH_FLIGHT}" == "ON" ]; then
+ python -c "import pyarrow.flight"
+ fi
Review comment:
```suggestion
if [ "${ARROW_S3}" == "ON" ]; then
python -c "import pyarrow._s3fs"
fi
if [ "${ARROW_CUDA}" == "ON" ]; then
python -c "import pyarrow.cuda"
fi
if [ "${ARROW_FLIGHT}" == "ON" ]; then
python -c "import pyarrow.flight"
fi
```
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
Review comment:
Why not just set `CMAKE_GENERATOR`?
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
Review comment:
```suggestion
-DARROW_BUILD_EXAMPLES=ON \
-DARROW_BUILD_INTEGRATION=ON \
-DARROW_BUILD_STATIC=OFF \
-DARROW_BUILD_TESTS=ON \
-DARROW_BUILD_UTILITIES=ON \
```
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_CUDA=${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-$DEFAULT_DEPENDENCY_SOURCE} \
+ -DARROW_FLIGHT=${ARROW_FLIGHT} \
+ -DARROW_GANDIVA=${ARROW_GANDIVA} \
+ -DARROW_HDFS=ON \
+ -DARROW_ORC=ON \
+ -DARROW_PARQUET=ON \
+ -DARROW_PLASMA=${ARROW_PLASMA} \
+ -DARROW_PYTHON=ON \
+ -DARROW_S3=${ARROW_S3} \
+ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-OFF} \
+ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
+ -DARROW_WITH_BROTLI=ON \
+ -DARROW_WITH_BZ2=ON \
+ -DARROW_WITH_LZ4=ON \
+ -DARROW_WITH_SNAPPY=ON \
+ -DARROW_WITH_ZLIB=ON \
+ -DARROW_WITH_ZSTD=ON \
+ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-release} \
+ -DCMAKE_INSTALL_LIBDIR=lib \
+ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
+ -DGTest_SOURCE=BUNDLED \
+ -DPARQUET_REQUIRE_ENCRYPTION=ON \
Review comment:
```suggestion
-DPARQUET_BUILD_EXAMPLES=ON \
-DPARQUET_BUILD_EXECUTABLES=ON \
-DPARQUET_REQUIRE_ENCRYPTION=ON \
```
##########
File path: dev/release/verify-release-candidate.sh
##########
@@ -378,22 +342,253 @@ test_csharp() {
fi
fi
- sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
- sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
+ CSHARP_ALREADY_INSTALLED=1
+}
+
+install_go() {
+ # Install go
+ if [ "${GO_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ local version=1.16.12
+
+ local arch="$(uname -m)"
+ if [ "$arch" == "x86_64" ]; then
+ arch=amd64
+ elif [ "$arch" == "aarch64" ]; then
+ arch=arm64
+ fi
+
+ if [ "$(uname)" == "Darwin" ]; then
+ local os=darwin
+ else
+ local os=linux
+ fi
+
+ local archive="go${version}.${os}-${arch}.tar.gz"
+ curl -sLO https://dl.google.com/go/$archive
+
+ local prefix=${ARROW_TMPDIR}/go
+ mkdir -p $prefix
+ tar -xzf $archive -C $prefix
+ rm -f $archive
+
+ export GOROOT=${prefix}/go
+ export GOPATH=${prefix}/gopath
+ export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
+
+ GO_ALREADY_INSTALLED=1
+}
+
+mamba() {
+ # Helper to access mamba directly instead after activating base conda environment
+ $ARROW_TMPDIR/mambaforge/bin/mamba "$@"
+}
+
+install_conda() {
+ # Setup short-lived miniconda for Python and integration tests
+ if [ "${CONDA_ALREADY_INSTALLED:-0}" -gt 0 ]; then
+ return 0
+ fi
+
+ local arch=$(uname -m)
+ local platform=$(uname)
+ local url="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-${platform}-${arch}.sh"
+ local prefix=$ARROW_TMPDIR/mambaforge
+
+ # Setup miniconda only if the directory doesn't exist yet
+ if [ ! -d "${prefix}" ]; then
+ curl -sL -o miniconda.sh $url
+ bash miniconda.sh -b -p $prefix
+ rm -f miniconda.sh
+ echo "Installed miniconda at ${prefix}"
+ else
+ echo "Miniconda already installed at ${prefix}"
+ fi
+
+ # Creating a separate conda environment
+ . $prefix/etc/profile.d/conda.sh
+ conda activate base
+
+ CONDA_ALREADY_INSTALLED=1
+}
+
+setup_conda() {
+ # Optionally setup conda environment with the passed dependencies
+ local env="conda-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ # Deactivate previous env
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ conda deactivate || :
+ fi
+ # Ensure that conda is installed
+ install_conda
+ # Create environment
+ if ! conda env list | grep $env; then
+ mamba create -y -n $env python=${pyver}
+ echo "Created conda environment ${CONDA_PREFIX}"
+ fi
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ mamba install -y -n $env $@
+ fi
+ # Activate the environment
+ conda activate $env
+ elif [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+}
+
+setup_virtualenv() {
+ # Optionally setup pip virtualenv with the passed dependencies
+ local env="venv-${ENV:-source}"
+ local pyver=${PYTHON_VERSION:-3}
+ local python=${PYTHON:-"python${pyver}"}
+ local virtualenv="${ARROW_TMPDIR}/${env}"
+ local skip_missing_python=${SKIP_MISSING_PYTHON:-0}
+
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ if [ ! -z ${CONDA_PREFIX} ]; then
+ echo "Conda environment is active despite that USE_CONDA is set to 0."
+ echo "Deactivate the environment before running the verification script."
+ exit 1
+ fi
+ # Deactivate previous env
+ deactivate || :
+ # Check that python interpreter exists
+ if ! command -v "${python}"; then
+ echo "Couldn't locate python interpreter with version ${pyver}"
+ echo "Call the script with USE_CONDA=1 to test all of the python versions."
+ if [ $skip_missing_python -gt 0 ]; then
+ continue
+ else
+ exit 1
+ fi
+ fi
+ # Create environment
+ if [ ! -d "${virtualenv}" ]; then
+ $python -m pip install virtualenv
+ $python -m virtualenv ${virtualenv}
+ fi
+ # Activate the environment
+ source "${virtualenv}/bin/activate"
+ # Install dependencies
+ if [ $# -gt 0 ]; then
+ pip install $@
+ fi
+ fi
+}
+
+setup_go() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_go
+ fi
+}
+
+setup_nodejs() {
+ if [ "${USE_CONDA}" -eq 0 ]; then
+ install_nodejs
+ fi
+}
+
+test_package_java() {
+ # Build and test Java (Requires newer Maven -- I used 3.3.9)
+ setup_conda maven
+
+ pushd java
+ mvn test
+ mvn package
popd
}
-# Build and test Python
+test_and_install_cpp() {
+ # Build and test C++
+ setup_virtualenv numpy
+ setup_conda \
+ --file ci/conda_env_unix.txt \
+ --file ci/conda_env_cpp.txt \
+ --file ci/conda_env_gandiva.txt \
+ ncurses \
+ numpy \
+ sqlite \
+ compilers
+
+ if [ "${USE_CONDA}" -gt 0 ]; then
+ DEFAULT_DEPENDENCY_SOURCE="CONDA"
+ else
+ DEFAULT_DEPENDENCY_SOURCE="AUTO"
+ fi
+
+ mkdir -p $ARROW_TMPDIR/build
+ pushd $ARROW_TMPDIR/build
+
+ if [ ! -z "$CMAKE_GENERATOR" ]; then
+ ARROW_CMAKE_OPTIONS="${ARROW_CMAKE_OPTIONS:-} -G Ninja"
+ fi
+
+ cmake \
+ -DARROW_BOOST_USE_SHARED=ON \
+ -DARROW_BUILD_INTEGRATION=ON \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_CUDA=${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-$DEFAULT_DEPENDENCY_SOURCE} \
+ -DARROW_FLIGHT=${ARROW_FLIGHT} \
Review comment:
```suggestion
-DARROW_FLIGHT=${ARROW_FLIGHT} \
-DARROW_FLIGHT_SQL=${ARROW_FLIGHT_SQL} \
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org