You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ap...@apache.org on 2022/08/17 09:00:22 UTC
[arrow] branch master updated: ARROW-17423: [CI][C++] Fix building CUDA docker images (#13896)
This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new f6127fca7a ARROW-17423: [CI][C++] Fix building CUDA docker images (#13896)
f6127fca7a is described below
commit f6127fca7ade9665f31493d37929346e651ed0e4
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Wed Aug 17 11:00:16 2022 +0200
ARROW-17423: [CI][C++] Fix building CUDA docker images (#13896)
* Update the CUDA runtime version as CUDA 9.1 images are not available anymore
* Fix passing child command arguments to "docker run"
Checked locally under a Ubuntu 20.04 host with:
```
UBUNTU=18.04 archery --debug docker run ubuntu-cuda-cpp
UBUNTU=20.04 archery --debug docker run ubuntu-cuda-cpp
```
Authored-by: Antoine Pitrou <an...@python.org>
Signed-off-by: Antoine Pitrou <an...@python.org>
---
.env | 2 +-
dev/archery/archery/docker/core.py | 18 ++++++------------
dev/archery/archery/docker/tests/test_docker.py | 2 +-
dev/archery/archery/integration/runner.py | 4 ++--
dev/archery/archery/integration/tester_cpp.py | 3 ++-
dev/archery/archery/integration/tester_csharp.py | 3 ++-
dev/archery/archery/integration/tester_java.py | 3 ++-
dev/archery/archery/integration/tester_js.py | 4 +++-
dev/archery/archery/integration/tester_rust.py | 3 ++-
dev/archery/archery/integration/util.py | 6 ------
dev/archery/archery/utils/source.py | 13 +++++++++++++
docker-compose.yml | 7 +++++++
12 files changed, 41 insertions(+), 27 deletions(-)
diff --git a/.env b/.env
index d754cbfca3..e160ffad90 100644
--- a/.env
+++ b/.env
@@ -54,7 +54,7 @@ UBUNTU=20.04
# Default versions for various dependencies
CLANG_TOOLS=12
-CUDA=9.1
+CUDA=11.0.3
DASK=latest
DOTNET=6.0
GCC_VERSION=""
diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py
index 4d6da095b8..600b7be600 100644
--- a/dev/archery/archery/docker/core.py
+++ b/dev/archery/archery/docker/core.py
@@ -16,7 +16,7 @@
# under the License.
import os
-import re
+import shlex
import subprocess
from io import StringIO
@@ -24,6 +24,7 @@ from dotenv import dotenv_values
from ruamel.yaml import YAML
from ..utils.command import Command, default_bin
+from ..utils.source import arrow_path
from ..compat import _ensure_path
@@ -42,12 +43,6 @@ def flatten(node, parents=None):
raise TypeError(node)
-def _sanitize_command(cmd):
- if isinstance(cmd, list):
- cmd = " ".join(cmd)
- return re.sub(r"\s+", " ", cmd)
-
-
_arch_short_mapping = {
'arm64v8': 'arm64',
}
@@ -294,7 +289,7 @@ class DockerCompose(Command):
args.extend([
'--output', 'type=docker',
- '-f', service['build']['dockerfile'],
+ '-f', arrow_path(service['build']['dockerfile']),
'-t', service['image'],
service['build'].get('context', '.')
])
@@ -306,7 +301,7 @@ class DockerCompose(Command):
for img in cache_from:
args.append('--cache-from="{}"'.format(img))
args.extend([
- '-f', service['build']['dockerfile'],
+ '-f', arrow_path(service['build']['dockerfile']),
'-t', service['image'],
service['build'].get('context', '.')
])
@@ -381,10 +376,9 @@ class DockerCompose(Command):
if command is not None:
args.append(command)
else:
- # replace whitespaces from the preformatted compose command
- cmd = _sanitize_command(service.get('command', ''))
+ cmd = service.get('command', '')
if cmd:
- args.append(cmd)
+ args.extend(shlex.split(cmd))
# execute as a plain docker cli command
self._execute_docker('run', '--rm', *args)
diff --git a/dev/archery/archery/docker/tests/test_docker.py b/dev/archery/archery/docker/tests/test_docker.py
index c268bb2312..899a0449e1 100644
--- a/dev/archery/archery/docker/tests/test_docker.py
+++ b/dev/archery/archery/docker/tests/test_docker.py
@@ -509,7 +509,7 @@ def test_image_with_gpu(arrow_compose_path):
"-e", "OTHER_ENV=2",
"-v", "/host:/container:rw",
"org/ubuntu-cuda",
- '/bin/bash -c "echo 1 > /tmp/dummy && cat /tmp/dummy"'
+ "/bin/bash", "-c", "echo 1 > /tmp/dummy && cat /tmp/dummy",
]
]
with assert_docker_calls(compose, expected_calls):
diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py
index 8a807497e7..0ea34b297c 100644
--- a/dev/archery/archery/integration/runner.py
+++ b/dev/archery/archery/integration/runner.py
@@ -35,8 +35,8 @@ from .tester_rust import RustTester
from .tester_java import JavaTester
from .tester_js import JSTester
from .tester_csharp import CSharpTester
-from .util import (ARROW_ROOT_DEFAULT, guid, SKIP_ARROW, SKIP_FLIGHT,
- printer)
+from .util import guid, SKIP_ARROW, SKIP_FLIGHT, printer
+from ..utils.source import ARROW_ROOT_DEFAULT
from . import datagen
diff --git a/dev/archery/archery/integration/tester_cpp.py b/dev/archery/archery/integration/tester_cpp.py
index d59198361b..52cc565dc0 100644
--- a/dev/archery/archery/integration/tester_cpp.py
+++ b/dev/archery/archery/integration/tester_cpp.py
@@ -20,7 +20,8 @@ import os
import subprocess
from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
_EXE_PATH = os.environ.get(
diff --git a/dev/archery/archery/integration/tester_csharp.py b/dev/archery/archery/integration/tester_csharp.py
index 144709f2e9..3f24d5f264 100644
--- a/dev/archery/archery/integration/tester_csharp.py
+++ b/dev/archery/archery/integration/tester_csharp.py
@@ -18,7 +18,8 @@
import os
from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
_EXE_PATH = os.path.join(
diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py
index 4c85a3a30b..45855079eb 100644
--- a/dev/archery/archery/integration/tester_java.py
+++ b/dev/archery/archery/integration/tester_java.py
@@ -20,7 +20,8 @@ import os
import subprocess
from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
def load_version_from_pom():
diff --git a/dev/archery/archery/integration/tester_js.py b/dev/archery/archery/integration/tester_js.py
index 80faefb8d8..6544a1fc6c 100644
--- a/dev/archery/archery/integration/tester_js.py
+++ b/dev/archery/archery/integration/tester_js.py
@@ -18,7 +18,9 @@
import os
from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
+
_EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'js/bin')
_VALIDATE = os.path.join(_EXE_PATH, 'integration.js')
diff --git a/dev/archery/archery/integration/tester_rust.py b/dev/archery/archery/integration/tester_rust.py
index 0e8943f2a8..c7a94de219 100644
--- a/dev/archery/archery/integration/tester_rust.py
+++ b/dev/archery/archery/integration/tester_rust.py
@@ -20,7 +20,8 @@ import os
import subprocess
from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
_EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, "rust/target/debug")
diff --git a/dev/archery/archery/integration/util.py b/dev/archery/archery/integration/util.py
index a4c4982ecb..80ba30052e 100644
--- a/dev/archery/archery/integration/util.py
+++ b/dev/archery/archery/integration/util.py
@@ -17,7 +17,6 @@
import contextlib
import io
-import os
import random
import socket
import subprocess
@@ -36,11 +35,6 @@ def guid():
SKIP_ARROW = 'arrow'
SKIP_FLIGHT = 'flight'
-ARROW_ROOT_DEFAULT = os.environ.get(
- 'ARROW_ROOT',
- os.path.abspath(__file__).rsplit("/", 5)[0]
-)
-
class _Printer:
"""
diff --git a/dev/archery/archery/utils/source.py b/dev/archery/archery/utils/source.py
index 1080cb75d6..e8f0ca92c4 100644
--- a/dev/archery/archery/utils/source.py
+++ b/dev/archery/archery/utils/source.py
@@ -22,6 +22,19 @@ import subprocess
from .git import git
+ARROW_ROOT_DEFAULT = os.environ.get(
+ 'ARROW_ROOT',
+ Path(__file__).resolve().parents[4]
+)
+
+
+def arrow_path(path):
+ """
+ Return full path to a file given its path inside the Arrow repo.
+ """
+ return os.path.join(ARROW_ROOT_DEFAULT, path)
+
+
class InvalidArrowSource(Exception):
pass
diff --git a/docker-compose.yml b/docker-compose.yml
index 903db17b5b..5755226a14 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -489,7 +489,14 @@ services:
ulimits: *ulimits
environment:
<<: *ccache
+ ARROW_BUILD_STATIC: "OFF"
ARROW_CUDA: "ON"
+ ARROW_GANDIVA: "OFF"
+ ARROW_GCS: "OFF"
+ ARROW_ORC: "OFF"
+ ARROW_S3: "OFF"
+ ARROW_SUBSTRAIT: "OFF"
+ ARROW_WITH_OPENTELEMETRY: "OFF"
volumes: *ubuntu-volumes
command: *cpp-command