You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@arrow.apache.org by "Raúl Cumplido (Jira)" <ji...@apache.org> on 2022/04/19 15:35:00 UTC
[jira] [Created] (ARROW-16236) [Python] [Packaging] test_s3fs_limited_permissions_create_bucket fails with Permission denied on MAC OS wheel builds
Raúl Cumplido created ARROW-16236:
-------------------------------------
Summary: [Python] [Packaging] test_s3fs_limited_permissions_create_bucket fails with Permission denied on MAC OS wheel builds
Key: ARROW-16236
URL: https://issues.apache.org/jira/browse/ARROW-16236
Project: Apache Arrow
Issue Type: Bug
Reporter: Raúl Cumplido
Fix For: 8.0.0
Nightly-packaging jobs for wheels on MAC OS have started failing with PermissionError:
Build example:
[https://github.com/ursacomputing/crossbow/runs/6074813901?check_suite_focus=true]
The error is related with the following S3 test
{code:java}
==================================== ERRORS ====================================
________ ERROR at setup of test_s3fs_limited_permissions_create_bucket _________request = <SubRequest 'limited_s3_user' for <Function test_s3fs_limited_permissions_create_bucket>>
s3_server = {'connection': ('localhost', 57493, 'arrow', 'apachearrow'), 'process': <Popen: returncode: None args: ['minio', '--compat', 'server', '--quiet', '-...>, 'tempdir': '/var/folders/cn/d0vdy6t94_77fwckh55bz2300000gq/T/tmpor9puihr'} @pytest.fixture(scope='session')
def limited_s3_user(request, s3_server):
if sys.platform == 'win32':
# Can't rely on FileNotFound check because
# there is sometimes an mc command on Windows
# which is unrelated to the minio mc
pytest.skip('The mc command is not installed on Windows')
request.config.pyarrow.requires('s3')
tempdir = s3_server['tempdir']
host, port, access_key, secret_key = s3_server['connection']
address = '{}:{}'.format(host, port)
> if not _configure_limited_user(tempdir, address, access_key, secret_key):test-arm64-env/lib/python3.9/site-packages/pyarrow/tests/test_fs.py:358:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
test-arm64-env/lib/python3.9/site-packages/pyarrow/tests/test_fs.py:317: in _configure_limited_user
if not _ensure_minio_component_version('mc', 2021):
test-arm64-env/lib/python3.9/site-packages/pyarrow/tests/test_fs.py:292: in _ensure_minio_component_version
proc = subprocess.Popen(full_args, stdout=subprocess.PIPE,
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/subprocess.py:951: in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Popen: returncode: 255 args: ['mc', '--version']>
args = ['mc', '--version'], executable = b'mc', preexec_fn = None
close_fds = True, pass_fds = (), cwd = None, env = None, startupinfo = None
creationflags = 0, shell = False, p2cread = -1, p2cwrite = -1, c2pread = 12
c2pwrite = 13, errread = 14, errwrite = 15, restore_signals = True, gid = None
gids = None, uid = None, umask = -1, start_new_session = False def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals,
gid, gids, uid, umask,
start_new_session):
"""Execute program (POSIX version)"""
if isinstance(args, (str, bytes)):
args = [args]
elif isinstance(args, os.PathLike):
if shell:
raise TypeError('path-like args is not allowed when '
'shell is true')
args = [args]
else:
args = list(args)
if shell:
# On Android the default shell is at '/system/bin/sh'.
unix_shell = ('/system/bin/sh' if
hasattr(sys, 'getandroidapilevel') else '/bin/sh')
args = [unix_shell, "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
sys.audit("subprocess.Popen", executable, args, cwd, env)
if (_USE_POSIX_SPAWN
and os.path.dirname(executable)
and preexec_fn is None
and not close_fds
and not pass_fds
and cwd is None
and (p2cread == -1 or p2cread > 2)
and (c2pwrite == -1 or c2pwrite > 2)
and (errwrite == -1 or errwrite > 2)
and not start_new_session
and gid is None
and gids is None
and uid is None
and umask < 0):
self._posix_spawn(args, executable, env, restore_signals,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
return
orig_executable = executable
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
errpipe_read, errpipe_write = os.pipe()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
# We must avoid complex work that could involve
# malloc or free in the child process to avoid
# potential deadlocks, thus we do all this here.
# and pass it to fork_exec()
if env is not None:
env_list = []
for k, v in env.items():
k = os.fsencode(k)
if b'=' in k:
raise ValueError("illegal environment variable name")
env_list.append(k + b'=' + os.fsencode(v))
else:
env_list = None # Use execv instead of execve.
executable = os.fsencode(executable)
if os.path.dirname(executable):
executable_list = (executable,)
else:
# This matches the behavior of os._execvpe().
executable_list = tuple(
os.path.join(os.fsencode(dir), executable)
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self.pid = _posixsubprocess.fork_exec(
args, executable_list,
close_fds, tuple(sorted(map(int, fds_to_keep))),
cwd, env_list,
p2cread, p2cwrite, c2pread, c2pwrite,
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session,
gid, gids, uid, umask,
preexec_fn)
self._child_created = True
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
self._close_pipe_fds(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
# Wait for exec to fail or succeed; possibly raising an
# exception (limited in size)
errpipe_data = bytearray()
while True:
part = os.read(errpipe_read, 50000)
errpipe_data += part
if not part or len(errpipe_data) > 50000:
break
finally:
# be sure the FD is closed no matter what
os.close(errpipe_read)
if errpipe_data:
try:
pid, sts = os.waitpid(self.pid, 0)
if pid == self.pid:
self._handle_exitstatus(sts)
else:
self.returncode = sys.maxsize
except ChildProcessError:
pass
try:
exception_name, hex_errno, err_msg = (
errpipe_data.split(b':', 2))
# The encoding here should match the encoding
# written in by the subprocess implementations
# like _posixsubprocess
err_msg = err_msg.decode()
except ValueError:
exception_name = b'SubprocessError'
hex_errno = b'0'
err_msg = 'Bad exception data from child: {!r}'.format(
bytes(errpipe_data))
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
SubprocessError)
if issubclass(child_exception_type, OSError) and hex_errno:
errno_num = int(hex_errno, 16)
child_exec_never_called = (err_msg == "noexec")
if child_exec_never_called:
err_msg = ""
# The error must be from chdir(cwd).
err_filename = cwd
else:
err_filename = orig_executable
if errno_num != 0:
err_msg = os.strerror(errno_num)
> raise child_exception_type(errno_num, err_msg, err_filename)
E PermissionError: [Errno 13] Permission denied: 'mc'
{code}
All the failed tasks with this error:
Failed Tasks:
- wheel-macos-big-sur-cp310-arm64:
URL: [https://github.com/ursacomputing/crossbow/tree/nightly-packaging-2022-04-19-0-github-wheel-macos-big-sur-cp310-arm64]
- wheel-macos-big-sur-cp310-universal2:
URL: [https://github.com/ursacomputing/crossbow/tree/nightly-packaging-2022-04-19-0-github-wheel-macos-big-sur-cp310-universal2]
- wheel-macos-big-sur-cp38-arm64:
URL: [https://github.com/ursacomputing/crossbow/tree/nightly-packaging-2022-04-19-0-github-wheel-macos-big-sur-cp38-arm64]
- wheel-macos-big-sur-cp39-arm64:
URL: [https://github.com/ursacomputing/crossbow/tree/nightly-packaging-2022-04-19-0-github-wheel-macos-big-sur-cp39-arm64]
--
This message was sent by Atlassian Jira
(v8.20.7#820007)