You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/06/14 22:57:36 UTC
[arrow] branch master updated: ARROW-12626: [C++] Support toolchain xsimd, update toolchain version to version 8.1.0 (#13244)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d63c16b741 ARROW-12626: [C++] Support toolchain xsimd, update toolchain version to version 8.1.0 (#13244)
d63c16b741 is described below
commit d63c16b741bcc71648c4f49d05b8abed79bc4007
Author: Wes McKinney <we...@users.noreply.github.com>
AuthorDate: Tue Jun 14 18:57:29 2022 -0400
ARROW-12626: [C++] Support toolchain xsimd, update toolchain version to version 8.1.0 (#13244)
This also updates pinned vcpkg to use xsimd 8.1.0.
This also implements auto python-wheel-windows-vs2017 image update mechanism.
We have a problem of "docker build" on Windows. "docker build" doesn't reuse pulled
image as caches. "docker build" always rebuilds an image. This implements manual
reuse mechanism like the following:
if ! docker pull; then
docker build # build only when built images don't exist
fi
docker run
But this doesn't work when ci/docker/python-wheel-windows-vs2017.dockerfile is updated
but pinned vcpkg revision isn't changed. In the case, "docker build" isn't run because
"docker pull" is succeeded.
To work this mechanism, this introduces "PYTHON_WHEEL_WINDOWS_IMAGE_REVISION".
We must bump it manually when we update ci/docker/python-wheel-windows-vs2017.dockerfile.
"PYTHON_WHEEL_WINDOWS_IMAGE_REVISION" is used in tag name. So "docker pull"
is failed with new "PYTHON_WHEEL_WINDOWS_IMAGE_REVISION" and then "docker build"
is used.
Lead-authored-by: Sutou Kouhei <ko...@clear-code.com>
Co-authored-by: Yibo Cai <yi...@arm.com>
Co-authored-by: Wes McKinney <we...@apache.org>
Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
.env | 13 ++++----
ci/conan/all/conanfile.py | 12 ++------
ci/conda_env_cpp.txt | 1 +
ci/docker/debian-10-cpp.dockerfile | 1 +
ci/docker/debian-11-cpp.dockerfile | 3 +-
ci/docker/fedora-35-cpp.dockerfile | 5 ++--
ci/docker/python-wheel-manylinux-201x.dockerfile | 3 +-
ci/docker/python-wheel-windows-vs2017.dockerfile | 3 ++
ci/docker/ubuntu-18.04-cpp.dockerfile | 1 +
ci/docker/ubuntu-20.04-cpp.dockerfile | 7 +++--
ci/docker/ubuntu-22.04-cpp.dockerfile | 5 ++--
ci/scripts/cpp_build.sh | 1 +
ci/scripts/python_wheel_macos_build.sh | 1 -
ci/scripts/python_wheel_manylinux_build.sh | 1 -
ci/scripts/python_wheel_windows_build.bat | 5 +++-
ci/vcpkg/ports.patch | 38 ------------------------
ci/vcpkg/vcpkg.json | 1 +
cpp/cmake_modules/ThirdpartyToolchain.cmake | 26 +++++++++++-----
cpp/thirdparty/versions.txt | 4 +--
cpp/vcpkg.json | 1 +
dev/archery/archery/docker/cli.py | 30 +++++++++++++++++++
dev/archery/archery/docker/core.py | 13 +++++---
dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 1 +
dev/tasks/conda-recipes/azure.osx.yml | 1 +
dev/tasks/macros.jinja | 11 ++++---
dev/tasks/python-wheels/github.osx.amd64.yml | 8 ++++-
dev/tasks/python-wheels/github.osx.arm64.yml | 6 ++--
dev/tasks/python-wheels/github.windows.yml | 27 +++++++++++++++--
docker-compose.yml | 16 ++--------
29 files changed, 144 insertions(+), 101 deletions(-)
diff --git a/.env b/.env
index 5345a27149..2f06cca474 100644
--- a/.env
+++ b/.env
@@ -93,12 +93,13 @@ DEVTOOLSET_VERSION=-1
# Please also update the crossbow configuration in order to keep the github
# actions cache up to date for the macOS wheels:
# https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-#
-# After commit 89295c9 openssl is only available as 3.0.2 which is
-# incompatible with aws-sdk-cpp 1.8.3 see
-# https://github.com/aws/aws-sdk-cpp/issues/1582 and causes issues on
-# manylinux2010
-VCPKG="89295c9"
+VCPKG="38bb87c"
+
+# This must be updated when we update
+# ci/docker/python-wheel-windows-vs2017.dockerfile.
+# This is a workaround for our CI problem that "archery docker build" doesn't
+# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12
# Use conanio/${CONAN} for "docker-compose run --rm conan". See
# https://github.com/conan-io/conan-docker-tools#readme for available
diff --git a/ci/conan/all/conanfile.py b/ci/conan/all/conanfile.py
index f34d6b6979..e0b325bfda 100644
--- a/ci/conan/all/conanfile.py
+++ b/ci/conan/all/conanfile.py
@@ -295,11 +295,7 @@ class ArrowConan(ConanFile):
if tools.Version(self.version) >= "6.0.0" and \
self.options.get_safe("simd_level") != None or \
self.options.get_safe("runtime_simd_level") != None:
- if tools.Version(self.version) >= "8.0.0":
- # TODO: Requires xsimd/master
- pass
- else:
- self.requires("xsimd/8.0.3")
+ self.requires("xsimd/8.1.0")
if self.options.with_zlib:
self.requires("zlib/1.2.12")
if self.options.with_zstd:
@@ -581,11 +577,7 @@ class ArrowConan(ConanFile):
if self.options.with_snappy:
self.cpp_info.components["libarrow"].requires.append("snappy::snappy")
if self.options.get_safe("simd_level") != None or self.options.get_safe("runtime_simd_level") != None:
- if tools.Version(self.version) >= "8.0.0":
- # Requires xsimd/master
- pass
- else:
- self.cpp_info.components["libarrow"].requires.append("xsimd::xsimd")
+ self.cpp_info.components["libarrow"].requires.append("xsimd::xsimd")
if self.options.with_zlib:
self.cpp_info.components["libarrow"].requires.append("zlib::zlib")
if self.options.with_zstd:
diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt
index 7120c56ebb..dd313f19d7 100644
--- a/ci/conda_env_cpp.txt
+++ b/ci/conda_env_cpp.txt
@@ -43,6 +43,7 @@ rapidjson
re2
snappy
thrift-cpp>=0.11.0
+xsimd
zlib
zstd
flatbuffers
diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile
index 15c85b868b..aae789164b 100644
--- a/ci/docker/debian-10-cpp.dockerfile
+++ b/ci/docker/debian-10-cpp.dockerfile
@@ -103,4 +103,5 @@ ENV ARROW_BUILD_TESTS=ON \
ORC_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED \
+ xsimd_SOURCE=BUNDLED \
zstd_SOURCE=BUNDLED
diff --git a/ci/docker/debian-11-cpp.dockerfile b/ci/docker/debian-11-cpp.dockerfile
index dfccd85e55..790a22dcd8 100644
--- a/ci/docker/debian-11-cpp.dockerfile
+++ b/ci/docker/debian-11-cpp.dockerfile
@@ -103,4 +103,5 @@ ENV ARROW_BUILD_TESTS=ON \
google_cloud_cpp_storage_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
- Protobuf_SOURCE=BUNDLED
+ Protobuf_SOURCE=BUNDLED \
+ xsimd_SOURCE=BUNDLED
diff --git a/ci/docker/fedora-35-cpp.dockerfile b/ci/docker/fedora-35-cpp.dockerfile
index 947c9aba1b..cbe9ee7539 100644
--- a/ci/docker/fedora-35-cpp.dockerfile
+++ b/ci/docker/fedora-35-cpp.dockerfile
@@ -95,6 +95,7 @@ ENV ARROW_BUILD_TESTS=ON \
CXX=g++ \
google_cloud_cpp_storage_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
- PARQUET_BUILD_EXECUTABLES=ON \
PARQUET_BUILD_EXAMPLES=ON \
- PATH=/usr/lib/ccache/:$PATH
+ PARQUET_BUILD_EXECUTABLES=ON \
+ PATH=/usr/lib/ccache/:$PATH \
+ xsimd_SOURCE=BUNDLED
diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
index d17ca0f85b..3a209f6cd2 100644
--- a/ci/docker/python-wheel-manylinux-201x.dockerfile
+++ b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -81,7 +81,8 @@ RUN vcpkg install \
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
- --x-feature=parquet
+ --x-feature=parquet \
+ --x-feature=s3
ARG python=3.8
ENV PYTHON_VERSION=${python}
diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
index 026c40b2af..247f13a15c 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -15,6 +15,9 @@
# specific language governing permissions and limitations
# under the License.
+# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
+# when you update this file.
+
# based on mcr.microsoft.com/windows/servercore:ltsc2019
# contains choco and vs2017 preinstalled
FROM abrarov/msvc-2017:2.11.0
diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile
index ed038f4c4c..16490845bd 100644
--- a/ci/docker/ubuntu-18.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-18.04-cpp.dockerfile
@@ -129,4 +129,5 @@ ENV ARROW_BUILD_TESTS=ON \
PATH=/usr/lib/ccache/:$PATH \
Thrift_SOURCE=BUNDLED \
utf8proc_SOURCE=BUNDLED \
+ xsimd_SOURCE=BUNDLED \
zstd_SOURCE=BUNDLED
diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile
index 0eade393cd..ae15835520 100644
--- a/ci/docker/ubuntu-20.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-20.04-cpp.dockerfile
@@ -150,11 +150,12 @@ ENV ARROW_BUILD_TESTS=ON \
ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
AWSSDK_SOURCE=BUNDLED \
google_cloud_cpp_storage_SOURCE=BUNDLED \
- GTest_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
+ GTest_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
PARQUET_BUILD_EXAMPLES=ON \
PARQUET_BUILD_EXECUTABLES=ON \
- PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED \
- PYTHON=python3
+ PATH=/usr/lib/ccache/:$PATH \
+ PYTHON=python3 \
+ xsimd_SOURCE=BUNDLED
diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile
index 1398dcd636..e7d2842dfc 100644
--- a/ci/docker/ubuntu-22.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -181,6 +181,7 @@ ENV ARROW_BUILD_TESTS=ON \
ORC_SOURCE=BUNDLED \
PARQUET_BUILD_EXAMPLES=ON \
PARQUET_BUILD_EXECUTABLES=ON \
- Protobuf_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
- PYTHON=python3
+ Protobuf_SOURCE=BUNDLED \
+ PYTHON=python3 \
+ xsimd_SOURCE=BUNDLED
diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh
index d88d95b677..8e823b0539 100755
--- a/ci/scripts/cpp_build.sh
+++ b/ci/scripts/cpp_build.sh
@@ -161,6 +161,7 @@ cmake \
-DThrift_SOURCE=${Thrift_SOURCE:-} \
-Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \
-Dzstd_SOURCE=${zstd_SOURCE:-} \
+ -Dxsimd_SOURCE=${xsimd_SOURCE:-} \
-G "${CMAKE_GENERATOR:-Ninja}" \
${CMAKE_ARGS} \
${source_dir}
diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh
index 7fa43a3eaa..a6e763b652 100755
--- a/ci/scripts/python_wheel_macos_build.sh
+++ b/ci/scripts/python_wheel_macos_build.sh
@@ -123,7 +123,6 @@ cmake \
-DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
-DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
-DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
- -DAWSSDK_SOURCE=BUNDLED \
-DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
-DCMAKE_INSTALL_LIBDIR=lib \
diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
index 6cfd34d851..af17606199 100755
--- a/ci/scripts/python_wheel_manylinux_build.sh
+++ b/ci/scripts/python_wheel_manylinux_build.sh
@@ -117,7 +117,6 @@ cmake \
-DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
-DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
-DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
- -DAWSSDK_SOURCE=BUNDLED \
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \
diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat
index 72072bf886..fb776185a5 100644
--- a/ci/scripts/python_wheel_windows_build.bat
+++ b/ci/scripts/python_wheel_windows_build.bat
@@ -47,6 +47,9 @@ set ARROW_WITH_LZ4=ON
set ARROW_WITH_SNAPPY=ON
set ARROW_WITH_ZLIB=ON
set ARROW_WITH_ZSTD=ON
+@rem Workaround for https://github.com/aws/aws-sdk-cpp/issues/1809 .
+@rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++.
+set AWSSDK_SOURCE=BUNDLED
set CMAKE_UNITY_BUILD=ON
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
set VCPKG_ROOT=C:\vcpkg
@@ -81,7 +84,7 @@ cmake ^
-DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
-DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
-DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
- -DAWSSDK_SOURCE=BUNDLED ^
+ -DAWSSDK_SOURCE=%AWSSDK_SOURCE% ^
-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
-DCMAKE_CXX_COMPILER=clcache ^
-DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch
index 83988cb405..b2eed47466 100644
--- a/ci/vcpkg/ports.patch
+++ b/ci/vcpkg/ports.patch
@@ -71,44 +71,6 @@ index 1289eed36a..b010a69f13 100644
)
vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
-diff --git a/ports/aws-sdk-cpp/fix-find-crypto.patch b/ports/aws-sdk-cpp/fix-find-crypto.patch
-new file mode 100644
-index 0000000000..aafc631e93
---- /dev/null
-+++ b/ports/aws-sdk-cpp/fix-find-crypto.patch
-@@ -0,0 +1,20 @@
-+diff --git a/cmake/setup_cmake_find_module.cmake b/cmake/setup_cmake_find_module.cmake
-+index f5f147a0f..4561b8c39 100644
-+--- a/cmake/setup_cmake_find_module.cmake
-++++ b/cmake/setup_cmake_find_module.cmake
-+@@ -30,15 +30,6 @@ file(WRITE
-+ "set(AWSSDK_INSTALL_BINDIR ${BINARY_DIRECTORY})\n"
-+ "set(AWSSDK_INSTALL_INCLUDEDIR ${INCLUDE_DIRECTORY})\n"
-+ "set(AWSSDK_INSTALL_ARCHIVEDIR ${ARCHIVE_DIRECTORY})\n"
-+- "if (NOT LibCrypto_INCLUDE_DIR)\n"
-+- " set(LibCrypto_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR} CACHE INTERNAL \"The OpenSSL include directory\")\n"
-+- "endif()\n"
-+- "if (NOT LibCrypto_STATIC_LIBRARY)\n"
-+- " set(LibCrypto_STATIC_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto static library\")\n"
-+- "endif()\n"
-+- "if (NOT LibCrypto_SHARED_LIBRARY)\n"
-+- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
-+- "endif()\n"
-+ )
-+
-+ if (NOT SIMPLE_INSTALL)
-diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
-index b520a17ae3..ea9e770cf5 100644
---- a/ports/aws-sdk-cpp/portfile.cmake
-+++ b/ports/aws-sdk-cpp/portfile.cmake
-@@ -8,6 +8,7 @@ vcpkg_from_github(
- PATCHES
- patch-relocatable-rpath.patch
- fix-aws-root.patch
-+ fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
index f81d0c491d..e5ea9cef57 100644
--- a/ports/curl/portfile.cmake
diff --git a/ci/vcpkg/vcpkg.json b/ci/vcpkg/vcpkg.json
index b62850631c..d9d074e99b 100644
--- a/ci/vcpkg/vcpkg.json
+++ b/ci/vcpkg/vcpkg.json
@@ -25,6 +25,7 @@
"re2",
"snappy",
"utf8proc",
+ "xsimd",
"zlib",
"zstd",
{
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 381b136e8d..6685ffb481 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -2204,16 +2204,20 @@ if((NOT ARROW_SIMD_LEVEL STREQUAL "NONE") OR (NOT ARROW_RUNTIME_SIMD_LEVEL STREQ
else()
set(ARROW_USE_XSIMD FALSE)
endif()
+
if(ARROW_USE_XSIMD)
- set(xsimd_SOURCE "BUNDLED")
- resolve_dependency(xsimd)
+ resolve_dependency(xsimd REQUIRED_VERSION "8.1.0")
- add_library(xsimd INTERFACE IMPORTED)
- if(CMAKE_VERSION VERSION_LESS 3.11)
- set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
- "${XSIMD_INCLUDE_DIR}")
+ if(xsimd_SOURCE STREQUAL "BUNDLED")
+ add_library(xsimd INTERFACE IMPORTED)
+ if(CMAKE_VERSION VERSION_LESS 3.11)
+ set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+ "${XSIMD_INCLUDE_DIR}")
+ else()
+ target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}")
+ endif()
else()
- target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}")
+ message(STATUS "xsimd found. Headers: ${xsimd_INCLUDE_DIRS}")
endif()
endif()
@@ -4437,6 +4441,14 @@ macro(build_awssdk)
-DENABLE_UNITY_BUILD=ON
"-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}"
"-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}")
+ if(NOT MSVC)
+ list(APPEND
+ AWSSDK_COMMON_CMAKE_ARGS
+ # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1582
+ "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations"
+ "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations"
+ )
+ endif()
# provide hint for AWS SDK to link with the already located openssl
get_filename_component(OPENSSL_ROOT_HINT "${OPENSSL_INCLUDE_DIR}" DIRECTORY)
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
index 69dc4486cd..589f7402ba 100644
--- a/cpp/thirdparty/versions.txt
+++ b/cpp/thirdparty/versions.txt
@@ -93,8 +93,8 @@ ARROW_UCX_BUILD_VERSION=1.12.1
ARROW_UCX_BUILD_SHA256_CHECKSUM=9bef31aed0e28bf1973d28d74d9ac4f8926c43ca3b7010bd22a084e164e31b71
ARROW_UTF8PROC_BUILD_VERSION=v2.7.0
ARROW_UTF8PROC_BUILD_SHA256_CHECKSUM=4bb121e297293c0fd55f08f83afab6d35d48f0af4ecc07523ad8ec99aa2b12a1
-ARROW_XSIMD_BUILD_VERSION=7d1778c3b38d63db7cec7145d939f40bc5d859d1
-ARROW_XSIMD_BUILD_SHA256_CHECKSUM=d861f4f3034550cdc62ad93fd60e8b0f2413a6ea49081c8698922b7b4f043ec6
+ARROW_XSIMD_BUILD_VERSION=8.1.0
+ARROW_XSIMD_BUILD_SHA256_CHECKSUM=d52551360d37709675237d2a0418e28f70995b5b7cdad7c674626bcfbbf48328
ARROW_ZLIB_BUILD_VERSION=1.2.12
ARROW_ZLIB_BUILD_SHA256_CHECKSUM=91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9
ARROW_ZSTD_BUILD_VERSION=v1.5.1
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
index 8e7fe75f5a..8e527adec3 100644
--- a/cpp/vcpkg.json
+++ b/cpp/vcpkg.json
@@ -47,6 +47,7 @@
"sqlite3",
"thrift",
"utf8proc",
+ "xsimd",
"zlib",
"zstd"
],
diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py
index c6b4a6473b..bbdd2261db 100644
--- a/dev/archery/archery/docker/cli.py
+++ b/dev/archery/archery/docker/cli.py
@@ -79,6 +79,36 @@ def check_config(obj):
# during the configuration loading
+@docker.command('pull')
+@click.argument('image')
+@click.option('--using-docker-cli', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_CLI',
+ help="Use docker CLI directly for pulling instead of calling "
+ "docker-compose. This may help to reuse cached layers.")
+@click.option('--pull-leaf/--no-leaf', default=True,
+ help="Whether to pull leaf images too.")
+@click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True,
+ help="Whether to ignore pull failures.")
+@click.pass_obj
+def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
+ ignore_pull_failures):
+ """
+ Execute docker-compose pull.
+ """
+ compose = obj['compose']
+
+ try:
+ compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli,
+ ignore_pull_failures=ignore_pull_failures)
+ except UndefinedImage as e:
+ raise click.ClickException(
+ "There is no service/image defined in docker-compose.yml with "
+ "name: {}".format(str(e))
+ )
+ except RuntimeError as e:
+ raise click.ClickException(str(e))
+
+
@docker.command('build')
@click.argument('image')
@click.option('--force-pull/--no-pull', default=True,
diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py
index dfdf3f4778..4d6da095b8 100644
--- a/dev/archery/archery/docker/core.py
+++ b/dev/archery/archery/docker/core.py
@@ -219,7 +219,8 @@ class DockerCompose(Command):
)
)
- def pull(self, service_name, pull_leaf=True, using_docker=False):
+ def pull(self, service_name, pull_leaf=True, using_docker=False,
+ ignore_pull_failures=True):
def _pull(service):
args = ['pull']
if service['image'] in self.pull_memory:
@@ -229,10 +230,14 @@ class DockerCompose(Command):
try:
self._execute_docker(*args, service['image'])
except Exception as e:
- # better --ignore-pull-failures handling
- print(e)
+ if ignore_pull_failures:
+ # better --ignore-pull-failures handling
+ print(e)
+ else:
+ raise
else:
- args.append('--ignore-pull-failures')
+ if ignore_pull_failures:
+ args.append('--ignore-pull-failures')
self._execute_compose(*args, service['name'])
self.pull_memory.add(service['image'])
diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
index 72fbda9c96..dcf1afe4a1 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
+++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
@@ -90,6 +90,7 @@ outputs:
- re2
- snappy
- thrift-cpp
+ - xsimd
- zlib
- zstd
run:
diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml
index 420b52c088..505a17cebe 100755
--- a/dev/tasks/conda-recipes/azure.osx.yml
+++ b/dev/tasks/conda-recipes/azure.osx.yml
@@ -57,6 +57,7 @@ jobs:
- script: |
source activate base
+ set -e
set +x
if [[ "${CONFIG}" == osx_arm* ]]; then
EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test"
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index b8e5669fa2..4e7fc4cf35 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -38,7 +38,7 @@ on:
{%- macro github_login_dockerhub() -%}
- name: Login to Dockerhub
- uses: docker/login-action@v1
+ uses: docker/login-action@v2
with:
username: {{ '${{ secrets.DOCKERHUB_USER }}' }}
password: {{ '${{ secrets.DOCKERHUB_TOKEN }}' }}
@@ -46,8 +46,11 @@ on:
{%- macro github_login_ghcr() -%}
- name: Login to GitHub Container Registry
- shell: bash
- run: docker login ghcr.io -u {{ '${{ github.repository_owner }}' }} -p {{ '${{ secrets.CROSSBOW_GHCR_TOKEN }}' }}
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: {{ '${{ github.actor }}' }}
+ password: {{ '${{ secrets.GITHUB_TOKEN }}' }}
{% endmacro %}
{%- macro github_install_archery() -%}
@@ -292,4 +295,4 @@ on:
# getwd() is necessary as this macro is used within jobs using a docker container
tools::write_PACKAGES(file.path(getwd(), "/repo/src/contrib", fsep = "/"), type = "source", verbose = TRUE)
- run: ls -R repo
-{% endmacro %}
\ No newline at end of file
+{% endmacro %}
diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index c18e080ac2..b647c3c7f4 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -74,6 +74,11 @@ jobs:
- name: Install Packages
run: |
+ if [ "${ARROW_S3}" == "ON" ]; then
+ x_feature_s3="--x-feature=s3"
+ else
+ x_feature_s3=
+ fi
vcpkg install \
--clean-after-build \
--x-install-root=${VCPKG_ROOT}/installed \
@@ -81,7 +86,8 @@ jobs:
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
- --x-feature=parquet
+ --x-feature=parquet \
+ ${x_feature_s3}
- name: Install Python {{ python_version }}
shell: bash
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 7198da7de4..f1ad9a7927 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -75,7 +75,8 @@ jobs:
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
- --x-feature=parquet
+ --x-feature=parquet \
+ --x-feature=s3
- name: Build ARM64 Wheel
env:
@@ -99,7 +100,8 @@ jobs:
--x-feature=flight \
--x-feature=gcs \
--x-feature=json \
- --x-feature=parquet
+ --x-feature=parquet \
+ --x-feature=s3
- name: Build AMD64 Wheel
env:
diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml
index ee4ebb96fb..6694e9feca 100644
--- a/dev/tasks/python-wheels/github.windows.yml
+++ b/dev/tasks/python-wheels/github.windows.yml
@@ -43,11 +43,34 @@ jobs:
- name: Build wheel
shell: cmd
- run: archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017
+ run: |
+ cd arrow
+ @rem We want to use only
+ @rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017
+ @rem but it doesn't use pulled caches.
+ @rem It always build an image from scratch.
+ @rem We can remove this workaround once we find a way to use
+ @rem pulled caches when build an image.
+ echo on
+ archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2017
+ if errorlevel 1 (
+ archery docker build --no-pull python-wheel-windows-vs2017 || exit /B 1
+ )
+ archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017
- name: Test wheel
shell: cmd
- run: archery docker run python-wheel-windows-test
+ run: |
+ cd arrow
+ archery docker run python-wheel-windows-test
{{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }}
{{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }}
+
+ {% if arrow.branch == 'master' %}
+ - name: Push Docker Image
+ shell: cmd
+ run: |
+ cd arrow
+ archery docker push python-wheel-windows-vs2017
+ {% endif %}
diff --git a/docker-compose.yml b/docker-compose.yml
index db7f4f1dfd..7a8ca192f9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -945,17 +945,7 @@ services:
command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow
python-wheel-windows-vs2017:
- # The windows images must be built locally and pushed to a remote registry:
- # export REPO=ghcr.io/ursacomputing/arrow
- # PYTHON=3.7 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017
- # PYTHON=3.8 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017
- # PYTHON=3.9 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017
- # PYTHON=3.10 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017
- # PYTHON=3.7 archery docker push python-wheel-windows-vs2017
- # PYTHON=3.8 archery docker push python-wheel-windows-vs2017
- # PYTHON=3.9 archery docker push python-wheel-windows-vs2017
- # PYTHON=3.10 archery docker push python-wheel-windows-vs2017
- image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}
+ image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
build:
args:
vcpkg: ${VCPKG}
@@ -965,8 +955,8 @@ services:
# This should make the pushed images reusable, but the image gets rebuilt.
# Uncomment if no local cache is available.
# cache_from:
- # - mcr.microsoft.com/windows/servercore:ltsc2019
- # - ${REPO}:wheel-windows-vs2017
+ # - abrarov/msvc-2017:2.11.0
+ # - ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
volumes:
- "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache"
- type: bind