You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@arrow.apache.org by "Joris Van den Bossche (Jira)" <ji...@apache.org> on 2020/12/16 09:08:01 UTC
[jira] [Assigned] (ARROW-10934) [Python] Tests are failed with
fsspec-0.8.5
[ https://issues.apache.org/jira/browse/ARROW-10934?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Joris Van den Bossche reassigned ARROW-10934:
---------------------------------------------
Assignee: Joris Van den Bossche
> [Python] Tests are failed with fsspec-0.8.5
> -------------------------------------------
>
> Key: ARROW-10934
> URL: https://issues.apache.org/jira/browse/ARROW-10934
> Project: Apache Arrow
> Issue Type: Improvement
> Components: Python
> Reporter: Kouhei Sutou
> Assignee: Joris Van den Bossche
> Priority: Blocker
> Labels: pull-request-available
> Fix For: 3.0.0
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/36851219/job/lwywl76d82coawpd?fullLog=true#L2284
> {noformat}
> ================================== FAILURES ===================================
> _ test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))] _
> fs = <pyarrow._fs.PyFileSystem object at 0x00000140F4C0A3A0>
> pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 0x00000140F4BFBB58>
> def test_get_file_info_with_selector(fs, pathfn):
> base_dir = pathfn('selector-dir/')
> file_a = pathfn('selector-dir/test_file_a')
> file_b = pathfn('selector-dir/test_file_b')
> dir_a = pathfn('selector-dir/test_dir_a')
> file_c = pathfn('selector-dir/test_dir_a/test_file_c')
> dir_b = pathfn('selector-dir/test_dir_b')
>
> try:
> fs.create_dir(base_dir)
> with fs.open_output_stream(file_a):
> pass
> with fs.open_output_stream(file_b):
> pass
> fs.create_dir(dir_a)
> with fs.open_output_stream(file_c):
> pass
> fs.create_dir(dir_b)
>
> # recursive selector
> selector = FileSelector(base_dir, allow_not_found=False,
> recursive=True)
> assert selector.base_dir == base_dir
>
> infos = fs.get_file_info(selector)
> if fs.type_name == "py::fsspec+s3":
> # s3fs only lists directories if they are not empty
> assert len(infos) == 4
> else:
> assert len(infos) == 5
>
> for info in infos:
> if (info.path.endswith(file_a) or info.path.endswith(file_b) or
> info.path.endswith(file_c)):
> assert info.type == FileType.File
> elif (info.path.rstrip("/").endswith(dir_a) or
> info.path.rstrip("/").endswith(dir_b)):
> assert info.type == FileType.Directory
> else:
> raise ValueError('unexpected path {}'.format(info.path))
> check_mtime_or_absent(info)
>
> # non-recursive selector -> not selecting the nested file_c
> selector = FileSelector(base_dir, recursive=False)
>
> infos = fs.get_file_info(selector)
> if fs.type_name == "py::fsspec+s3":
> # s3fs only lists directories if they are not empty
> assert len(infos) == 3
> else:
> assert len(infos) == 4
>
> finally:
> > fs.delete_dir(base_dir)
> pyarrow\tests\test_fs.py:716:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> pyarrow\_fs.pyx:472: in pyarrow._fs.FileSystem.delete_dir
> check_status(self.fs.DeleteDir(directory))
> pyarrow\_fs.pyx:1035: in pyarrow._fs._cb_delete_dir
> handler.delete_dir(frombytes(path))
> pyarrow\fs.py:262: in delete_dir
> self.fs.rm(path, recursive=True)
> C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:176: in rm
> self.rm_file(p)
> C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\spec.py:840: in rm_file
> self._rm(path)
> C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:163: in _rm
> self.rmdir(path)
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> self = <fsspec.implementations.memory.MemoryFileSystem object at 0x00000140F4ABBC58>
> path = 'selector-dir'
> def rmdir(self, path):
> path = path.rstrip("/")
> if path in self.pseudo_dirs:
> if not self.ls(path):
> self.pseudo_dirs.remove(path)
> else:
> > raise OSError(ENOTEMPTY, "Directory not empty", path)
> E OSError: [Errno 41] Directory not empty: 'selector-dir'
> C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:110: OSError
> __ test_delete_dir[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))] __
> fs = <pyarrow._fs.PyFileSystem object at 0x00000140F4C1CDC0>
> pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 0x00000140F50BC738>
> def test_delete_dir(fs, pathfn):
> skip_fsspec_s3fs(fs)
>
> d = pathfn('directory/')
> nd = pathfn('directory/nested/')
>
> fs.create_dir(nd)
> > fs.delete_dir(d)
> pyarrow\tests\test_fs.py:743:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> pyarrow\_fs.pyx:472: in pyarrow._fs.FileSystem.delete_dir
> check_status(self.fs.DeleteDir(directory))
> pyarrow\_fs.pyx:1035: in pyarrow._fs._cb_delete_dir
> handler.delete_dir(frombytes(path))
> pyarrow\fs.py:262: in delete_dir
> self.fs.rm(path, recursive=True)
> C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:176: in rm
> self.rm_file(p)
> C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\spec.py:840: in rm_file
> self._rm(path)
> C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:163: in _rm
> self.rmdir(path)
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> self = <fsspec.implementations.memory.MemoryFileSystem object at 0x00000140F4ABBC58>
> path = 'directory'
> def rmdir(self, path):
> path = path.rstrip("/")
> if path in self.pseudo_dirs:
> if not self.ls(path):
> self.pseudo_dirs.remove(path)
> else:
> > raise OSError(ENOTEMPTY, "Directory not empty", path)
> E OSError: [Errno 41] Directory not empty: 'directory'
> C:\Miniconda37-x64\envs\arrow\lib\site-packages\fsspec\implementations\memory.py:110: OSError
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)