You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2025/01/01 14:06:10 UTC
(arrow) branch main updated: GH-45137: [CI][C++] Add a GCC 15 job (#45138)
This is an automated email from the ASF dual-hosted git repository.
jonkeane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 0b6a687fa8 GH-45137: [CI][C++] Add a GCC 15 job (#45138)
0b6a687fa8 is described below
commit 0b6a687fa826ce9d9db905aaff42e176062428e2
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Wed Jan 1 23:06:03 2025 +0900
GH-45137: [CI][C++] Add a GCC 15 job (#45138)
### Rationale for this change
GCC 15 isn't released yet but we'll be able to add support for the latest GCC in a timely manner with this.
### What changes are included in this PR?
Add a GCC 15 job based on Debian GNU/Linux experimental.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No.
* GitHub Issue: #45137
Authored-by: Sutou Kouhei <ko...@clear-code.com>
Signed-off-by: Jonathan Keane <jk...@gmail.com>
---
.env | 2 +-
ci/docker/debian-experimental-cpp.dockerfile | 143 +++++++++++++++++++++++++++
ci/docker/linux-apt-r.dockerfile | 8 +-
ci/docker/ubuntu-22.04-cpp.dockerfile | 21 ++--
ci/docker/ubuntu-24.04-cpp.dockerfile | 21 ++--
dev/tasks/tasks.yml | 25 ++++-
docker-compose.yml | 17 ++--
7 files changed, 193 insertions(+), 44 deletions(-)
diff --git a/.env b/.env
index 0b1a988875..859672e733 100644
--- a/.env
+++ b/.env
@@ -57,7 +57,7 @@ CLANG_TOOLS=14
CUDA=11.2.2
DASK=latest
DOTNET=8.0
-GCC_VERSION=""
+GCC=
HDFS=3.2.1
JDK=11
KARTOTHEK=latest
diff --git a/ci/docker/debian-experimental-cpp.dockerfile b/ci/docker/debian-experimental-cpp.dockerfile
new file mode 100644
index 0000000000..2721b1d5f2
--- /dev/null
+++ b/ci/docker/debian-experimental-cpp.dockerfile
@@ -0,0 +1,143 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG arch=amd64
+FROM ${arch}/debian:experimental
+ARG arch
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ARG gcc
+ARG llvm
+RUN if [ -n "${gcc}" ]; then \
+ gcc_package_suffix="-${gcc}"; \
+ else \
+ gcc_package_suffix=""; \
+ fi && \
+ if [ -n "${llvm}" ]; then \
+ llvm_package_suffix="-${llvm}"; \
+ else \
+ llvm_package_suffix=""; \
+ fi && \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ autoconf \
+ ccache \
+ cmake \
+ curl \
+ g++ \
+ gcc \
+ gdb \
+ git \
+ libbenchmark-dev \
+ libboost-filesystem-dev \
+ libboost-system-dev \
+ libbrotli-dev \
+ libbz2-dev \
+ libc-ares-dev \
+ libcurl4-openssl-dev \
+ libgflags-dev \
+ libgmock-dev \
+ libgoogle-glog-dev \
+ libgrpc++-dev \
+ libidn2-dev \
+ libkrb5-dev \
+ libldap-dev \
+ liblz4-dev \
+ libnghttp2-dev \
+ libprotobuf-dev \
+ libprotoc-dev \
+ libpsl-dev \
+ libre2-dev \
+ librtmp-dev \
+ libsnappy-dev \
+ libsqlite3-dev \
+ libssh-dev \
+ libssh2-1-dev \
+ libssl-dev \
+ libthrift-dev \
+ libutf8proc-dev \
+ libxml2-dev \
+ libxsimd-dev \
+ libzstd-dev \
+ make \
+ ninja-build \
+ nlohmann-json3-dev \
+ npm \
+ opentelemetry-cpp-dev \
+ pkg-config \
+ protobuf-compiler-grpc \
+ python3-dev \
+ python3-pip \
+ python3-venv \
+ rapidjson-dev \
+ rsync \
+ tzdata \
+ zlib1g-dev && \
+ apt-get install -y -q --no-install-recommends -t experimental \
+ clang${llvm_package_suffix} \
+ g++${gcc_package_suffix} \
+ gcc${gcc_package_suffix} \
+ llvm${llvm_package_suffix}-dev && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+
+COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_gcs_testbench.sh default
+
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
+COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
+
+# Prioritize system packages and local installation.
+ENV ARROW_ACERO=ON \
+ ARROW_AZURE=ON \
+ ARROW_BUILD_TESTS=ON \
+ ARROW_DATASET=ON \
+ ARROW_DEPENDENCY_SOURCE=SYSTEM \
+ ARROW_DATASET=ON \
+ ARROW_FLIGHT=ON \
+ ARROW_FLIGHT_SQL=ON \
+ ARROW_GANDIVA=ON \
+ ARROW_GCS=ON \
+ ARROW_HOME=/usr/local \
+ ARROW_JEMALLOC=ON \
+ ARROW_ORC=ON \
+ ARROW_PARQUET=ON \
+ ARROW_S3=ON \
+ ARROW_SUBSTRAIT=ON \
+ ARROW_USE_CCACHE=ON \
+ ARROW_WITH_BROTLI=ON \
+ ARROW_WITH_BZ2=ON \
+ ARROW_WITH_LZ4=ON \
+ ARROW_WITH_OPENTELEMETRY=ON \
+ ARROW_WITH_SNAPPY=ON \
+ ARROW_WITH_ZLIB=ON \
+ ARROW_WITH_ZSTD=ON \
+ AWSSDK_SOURCE=BUNDLED \
+ Azure_SOURCE=BUNDLED \
+ CC=gcc${gcc:+-${gcc}} \
+ CXX=g++${gcc:+-${gcc}} \
+ google_cloud_cpp_storage_SOURCE=BUNDLED \
+ ORC_SOURCE=BUNDLED \
+ PATH=/usr/lib/ccache/:$PATH \
+ PYTHON=python3
diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile
index 4be5adf246..48c7154ef0 100644
--- a/ci/docker/linux-apt-r.dockerfile
+++ b/ci/docker/linux-apt-r.dockerfile
@@ -65,10 +65,10 @@ RUN apt-get update -y && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
-ARG gcc_version=""
-RUN if [ "${gcc_version}" != "" ]; then \
- update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
- update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
+ARG gcc=""
+RUN if [ "${gcc}" != "" ]; then \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc} 100 && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc} 100 && \
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 && \
update-alternatives --set cc /usr/bin/gcc && \
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 && \
diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile
index 2e4d658bf9..28cef29463 100644
--- a/ci/docker/ubuntu-22.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -137,32 +137,27 @@ RUN cd ~ && git clone https://github.com/emscripten-core/emsdk.git && \
echo "Installed emsdk to:" ~/emsdk
-ARG gcc_version=""
-RUN if [ "${gcc_version}" = "" ]; then \
+ARG gcc=""
+RUN if [ "${gcc}" = "" ]; then \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
g++ \
gcc; \
else \
- if [ "${gcc_version}" -gt "12" ]; then \
- apt-get update -y -q && \
- apt-get install -y -q --no-install-recommends software-properties-common && \
- add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
- fi; \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
- g++-${gcc_version} \
- gcc-${gcc_version} && \
- update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
- update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
+ g++-${gcc} \
+ gcc-${gcc} && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc} 100 && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc} 100 && \
update-alternatives --install \
/usr/bin/$(uname --machine)-linux-gnu-gcc \
$(uname --machine)-linux-gnu-gcc \
- /usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc_version} 100 && \
+ /usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc} 100 && \
update-alternatives --install \
/usr/bin/$(uname --machine)-linux-gnu-g++ \
$(uname --machine)-linux-gnu-g++ \
- /usr/bin/$(uname --machine)-linux-gnu-g++-${gcc_version} 100 && \
+ /usr/bin/$(uname --machine)-linux-gnu-g++-${gcc} 100 && \
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
update-alternatives --set cc /usr/bin/gcc && \
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \
diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile
index 53113bccfe..3f486b09f9 100644
--- a/ci/docker/ubuntu-24.04-cpp.dockerfile
+++ b/ci/docker/ubuntu-24.04-cpp.dockerfile
@@ -128,32 +128,27 @@ RUN apt-get update -y -q && \
apt-get clean && \
rm -rf /var/lib/apt/lists*
-ARG gcc_version=""
-RUN if [ "${gcc_version}" = "" ]; then \
+ARG gcc=""
+RUN if [ "${gcc}" = "" ]; then \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
g++ \
gcc; \
else \
- if [ "${gcc_version}" -gt "14" ]; then \
- apt-get update -y -q && \
- apt-get install -y -q --no-install-recommends software-properties-common && \
- add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
- fi; \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
- g++-${gcc_version} \
- gcc-${gcc_version} && \
- update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
- update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
+ g++-${gcc} \
+ gcc-${gcc} && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc} 100 && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc} 100 && \
update-alternatives --install \
/usr/bin/$(uname --machine)-linux-gnu-gcc \
$(uname --machine)-linux-gnu-gcc \
- /usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc_version} 100 && \
+ /usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc} 100 && \
update-alternatives --install \
/usr/bin/$(uname --machine)-linux-gnu-g++ \
$(uname --machine)-linux-gnu-g++ \
- /usr/bin/$(uname --machine)-linux-gnu-g++-${gcc_version} 100 && \
+ /usr/bin/$(uname --machine)-linux-gnu-g++-${gcc} 100 && \
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
update-alternatives --set cc /usr/bin/gcc && \
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index c8c311f513..9f04d33f83 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -74,9 +74,13 @@ groups:
test:
- test-*
+ # Can be removed after we improved C++20 support
+ - ~test-debian-experimental-cpp-gcc-15
cpp:
- test-*cpp*
+ # Can be removed after we improved C++20 support
+ - ~test-debian-experimental-cpp-gcc-15
- example-*cpp*
c-glib:
@@ -157,6 +161,8 @@ groups:
nightly-tests:
- test-*
+ # Can be removed after we improved C++20 support
+ - ~test-debian-experimental-cpp-gcc-15
- example-*
nightly-packaging:
@@ -935,7 +941,7 @@ tasks:
params:
env:
UBUNTU: 24.04
- GCC_VERSION: 13
+ GCC: 13
image: ubuntu-cpp-bundled
test-ubuntu-24.04-cpp:
@@ -954,7 +960,7 @@ tasks:
params:
env:
CLANG_TOOLS: 15
- GCC_VERSION: 14
+ GCC: 14
LLVM: 15
UBUNTU: 24.04
# rapidjson 1.1.0 has an error caught by gcc 14.
@@ -997,6 +1003,17 @@ tasks:
image: debian-cpp
{% endfor %}
+ test-debian-experimental-cpp-gcc-15:
+ ci: github
+ template: docker-tests/github.linux.yml
+ params:
+ env:
+ ARCH: "amd64"
+ DEBIAN: "experimental"
+ GCC: "15"
+ flags: "-e CMAKE_CXX_STANDARD=20"
+ image: debian-cpp
+
test-fedora-39-cpp:
ci: github
template: docker-tests/github.linux.yml
@@ -1301,7 +1318,7 @@ tasks:
params:
env:
UBUNTU: 22.04
- GCC_VERSION: 11
+ GCC: 11
image: ubuntu-r-only-r
# This also has -flto=auto
@@ -1311,7 +1328,7 @@ tasks:
params:
env:
UBUNTU: 22.04
- GCC_VERSION: 12
+ GCC: 12
image: ubuntu-r-only-r
test-r-minimal-build:
diff --git a/docker-compose.yml b/docker-compose.yml
index 7aabbb43b4..51d5951ad6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -188,8 +188,6 @@ volumes:
name: ${ARCH}-conda-ccache
debian-ccache:
name: ${ARCH}-debian-${DEBIAN}-ccache
- debian-rust:
- name: ${ARCH}-debian-${DEBIAN}-rust
fedora-ccache:
name: ${ARCH}-fedora-${FEDORA}-ccache
maven-cache:
@@ -346,7 +344,7 @@ services:
# docker compose run --rm debian-cpp
# Parameters:
# ARCH: amd64, arm64v8, ...
- # DEBIAN: 12
+ # DEBIAN: 12, experimental
image: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
build:
context: .
@@ -355,6 +353,7 @@ services:
- ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
args:
arch: ${ARCH}
+ gcc: ${GCC}
llvm: ${LLVM}
shm_size: *shm-size
ulimits: *ulimits
@@ -389,8 +388,8 @@ services:
arch: ${ARCH}
base: "${ARCH}/ubuntu:${UBUNTU}"
clang_tools: ${CLANG_TOOLS}
+ gcc: ${GCC}
llvm: ${LLVM}
- gcc_version: ${GCC_VERSION}
shm_size: *shm-size
cap_add:
- SYS_ADMIN
@@ -425,8 +424,8 @@ services:
arch: ${ARCH}
base: "${ARCH}/ubuntu:${UBUNTU}"
clang_tools: ${CLANG_TOOLS}
+ gcc: ${GCC}
llvm: ${LLVM}
- gcc_version: ${GCC_VERSION}
shm_size: *shm-size
cap_add:
- SYS_ADMIN
@@ -1517,12 +1516,12 @@ services:
- ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R}
args:
arch: ${ARCH}
- r: ${R}
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
- gcc_version: ${GCC_VERSION}
- tz: ${TZ}
- r_prune_deps: ${R_PRUNE_DEPS}
+ gcc: ${GCC}
+ r: ${R}
r_duckdb_dev: ${R_DUCKDB_DEV:-}
+ r_prune_deps: ${R_PRUNE_DEPS}
+ tz: ${TZ}
shm_size: *shm-size
environment:
<<: [*common, *ccache, *sccache]