You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@arrow.apache.org by "Kouhei Sutou (Jira)" <ji...@apache.org> on 2020/12/16 03:24:00 UTC
[jira] [Created] (ARROW-10934) [Python] Tests are failed with
fsspec-0.8.5
Kouhei Sutou created ARROW-10934:
------------------------------------
Summary: [Python] Tests are failed with fsspec-0.8.5
Key: ARROW-10934
URL: https://issues.apache.org/jira/browse/ARROW-10934
Project: Apache Arrow
Issue Type: Improvement
Components: Python
Reporter: Kouhei Sutou
Fix For: 3.0.0
https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/36851219/job/lwywl76d82coawpd?fullLog=true#L2284
{noformat}
================================== FAILURES ===================================
_ test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))] _
fs = <pyarrow._fs.PyFileSystem object at 0x00000140F4C0A3A0>
pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 0x00000140F4BFBB58>
def test_get_file_info_with_selector(fs, pathfn):
base_dir = pathfn('selector-dir/')
file_a = pathfn('selector-dir/test_file_a')
file_b = pathfn('selector-dir/test_file_b')
dir_a = pathfn('selector-dir/test_dir_a')
file_c = pathfn('selector-dir/test_dir_a/test_file_c')
dir_b = pathfn('selector-dir/test_dir_b')
try:
fs.create_dir(base_dir)
with fs.open_output_stream(file_a):
pass
with fs.open_output_stream(file_b):
pass
fs.create_dir(dir_a)
with fs.open_output_stream(file_c):
pass
fs.create_dir(dir_b)
# recursive selector
selector = FileSelector(base_dir, allow_not_found=False,
recursive=True)
assert selector.base_dir == base_dir
infos = fs.get_file_info(selector)
if fs.type_name == "py::fsspec+s3":
# s3fs only lists directories if they are not empty
assert len(infos) == 4
else:
assert len(infos) == 5
for info in infos:
if (info.path.endswith(file_a) or info.path.endswith(file_b) or
info.path.endswith(file_c)):
assert info.type == FileType.File
elif (info.path.rstrip("/").endswith(dir_a) or
info.path.rstrip("/").endswith(dir_b)):
assert info.type == FileType.Directory
else:
raise ValueError('unexpected path {}'.format(info.path))
check_mtime_or_absent(info)
# non-recursive selector -> not selecting the nested file_c
selector = FileSelector(base_dir, recursive=False)
infos = fs.get_file_info(selector)
if fs.type_name == "py::fsspec+s3":
# s3fs only lists directories if they are not empty
assert len(infos) == 3
else:
assert len(infos) == 4
finally:
> fs.delete_dir(base_dir)
pyarrow\tests\test_fs.py:716:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
pyarrow\_fs.pyx:472: in pyarrow._fs.FileSystem.delete_dir
check_status(self.fs.DeleteDir(directory))
pyarrow\_fs.pyx:1035: in pyarrow._fs._cb_delete_dir
handler.delete_dir(frombytes(path))
pyarrow\fs.py:262: in delete_dir
self.fs.rm(path, recursive=True)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:176: in rm
self.rm_file(p)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\spec.py:840: in rm_file
self._rm(path)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:163: in _rm
self.rmdir(path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <fsspec.implementations.memory.MemoryFileSystem object at 0x00000140F4ABBC58>
path = 'selector-dir'
def rmdir(self, path):
path = path.rstrip("/")
if path in self.pseudo_dirs:
if not self.ls(path):
self.pseudo_dirs.remove(path)
else:
> raise OSError(ENOTEMPTY, "Directory not empty", path)
E OSError: [Errno 41] Directory not empty: 'selector-dir'
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:110: OSError
__ test_delete_dir[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))] __
fs = <pyarrow._fs.PyFileSystem object at 0x00000140F4C1CDC0>
pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 0x00000140F50BC738>
def test_delete_dir(fs, pathfn):
skip_fsspec_s3fs(fs)
d = pathfn('directory/')
nd = pathfn('directory/nested/')
fs.create_dir(nd)
> fs.delete_dir(d)
pyarrow\tests\test_fs.py:743:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
pyarrow\_fs.pyx:472: in pyarrow._fs.FileSystem.delete_dir
check_status(self.fs.DeleteDir(directory))
pyarrow\_fs.pyx:1035: in pyarrow._fs._cb_delete_dir
handler.delete_dir(frombytes(path))
pyarrow\fs.py:262: in delete_dir
self.fs.rm(path, recursive=True)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:176: in rm
self.rm_file(p)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\spec.py:840: in rm_file
self._rm(path)
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:163: in _rm
self.rmdir(path)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <fsspec.implementations.memory.MemoryFileSystem object at 0x00000140F4ABBC58>
path = 'directory'
def rmdir(self, path):
path = path.rstrip("/")
if path in self.pseudo_dirs:
if not self.ls(path):
self.pseudo_dirs.remove(path)
else:
> raise OSError(ENOTEMPTY, "Directory not empty", path)
E OSError: [Errno 41] Directory not empty: 'directory'
C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:110: OSError
{noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)