You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@beam.apache.org by "Brian Hulette (Jira)" <ji...@apache.org> on 2020/10/15 18:32:00 UTC
[jira] [Reopened] (BEAM-10921) Python unit tests on windows flaky
[ https://issues.apache.org/jira/browse/BEAM-10921?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Brian Hulette reopened BEAM-10921:
----------------------------------
It looks like this is happening again: [https://github.com/apache/beam/pull/13102/checks?check_run_id=1256599103]
{code}
================================== FAILURES ===================================
__________________ ElementStreamTest.test_done_if_terminated __________________
[gw0] win32 -- Python 3.7.9 d:\a\beam\beam\sdks\python\target\.tox\py37-win\scripts\python.exe
self = <apache_beam.runners.interactive.recording_manager_test.ElementStreamTest testMethod=test_done_if_terminated>
def setUp(self):
> ie.new_env()
apache_beam\runners\interactive\recording_manager_test.py:75:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam\runners\interactive\interactive_environment.py:118: in new_env
_interactive_beam_env.cleanup()
apache_beam\runners\interactive\interactive_environment.py:272: in cleanup
cache_manager.cleanup()
apache_beam\runners\interactive\caching\streaming_cache.py:391: in cleanup
shutil.rmtree(self._cache_dir)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:516: in rmtree
return _rmtree_unsafe(path, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:395: in _rmtree_unsafe
_rmtree_unsafe(fullname, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:400: in _rmtree_unsafe
onerror(os.unlink, fullname, sys.exc_info())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
path = 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-4izmxjl32526131575432\\full'
onerror = <function rmtree.<locals>.onerror at 0x0000024C29ADC948>
def _rmtree_unsafe(path, onerror):
try:
with os.scandir(path) as scandir_it:
entries = list(scandir_it)
except OSError:
onerror(os.scandir, path, sys.exc_info())
entries = []
for entry in entries:
fullname = entry.path
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError:
is_dir = False
if is_dir:
try:
if entry.is_symlink():
# This can only happen if someone replaces
# a directory with a symlink after the call to
# os.scandir or entry.is_dir above.
raise OSError("Cannot call rmtree on a symbolic link")
except OSError:
onerror(os.path.islink, fullname, sys.exc_info())
continue
_rmtree_unsafe(fullname, onerror)
else:
try:
> os.unlink(fullname)
E PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-4izmxjl32526131575432\\full\\c24ca9d948-2526133224200-2526136692168-2526131575432'
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:398: PermissionError
_________________________ ElementStreamTest.test_read _________________________
[gw0] win32 -- Python 3.7.9 d:\a\beam\beam\sdks\python\target\.tox\py37-win\scripts\python.exe
self = <apache_beam.runners.interactive.recording_manager_test.ElementStreamTest testMethod=test_read>
def setUp(self):
> ie.new_env()
apache_beam\runners\interactive\recording_manager_test.py:75:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam\runners\interactive\interactive_environment.py:118: in new_env
_interactive_beam_env.cleanup()
apache_beam\runners\interactive\interactive_environment.py:272: in cleanup
cache_manager.cleanup()
apache_beam\runners\interactive\caching\streaming_cache.py:391: in cleanup
shutil.rmtree(self._cache_dir)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:516: in rmtree
return _rmtree_unsafe(path, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:395: in _rmtree_unsafe
_rmtree_unsafe(fullname, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:400: in _rmtree_unsafe
onerror(os.unlink, fullname, sys.exc_info())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
path = 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-4izmxjl32526131575432\\full'
onerror = <function rmtree.<locals>.onerror at 0x0000024C2A51C4C8>
def _rmtree_unsafe(path, onerror):
try:
with os.scandir(path) as scandir_it:
entries = list(scandir_it)
except OSError:
onerror(os.scandir, path, sys.exc_info())
entries = []
for entry in entries:
fullname = entry.path
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError:
is_dir = False
if is_dir:
try:
if entry.is_symlink():
# This can only happen if someone replaces
# a directory with a symlink after the call to
# os.scandir or entry.is_dir above.
raise OSError("Cannot call rmtree on a symbolic link")
except OSError:
onerror(os.path.islink, fullname, sys.exc_info())
continue
_rmtree_unsafe(fullname, onerror)
else:
try:
> os.unlink(fullname)
E PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-4izmxjl32526131575432\\full\\c24ca9d948-2526133224200-2526136692168-2526131575432'
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:398: PermissionError
____________________ ElementStreamTest.test_read_duration _____________________
[gw0] win32 -- Python 3.7.9 d:\a\beam\beam\sdks\python\target\.tox\py37-win\scripts\python.exe
self = <apache_beam.runners.interactive.recording_manager_test.ElementStreamTest testMethod=test_read_duration>
def setUp(self):
> ie.new_env()
apache_beam\runners\interactive\recording_manager_test.py:75:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam\runners\interactive\interactive_environment.py:118: in new_env
_interactive_beam_env.cleanup()
apache_beam\runners\interactive\interactive_environment.py:272: in cleanup
cache_manager.cleanup()
apache_beam\runners\interactive\caching\streaming_cache.py:391: in cleanup
shutil.rmtree(self._cache_dir)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:516: in rmtree
return _rmtree_unsafe(path, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:395: in _rmtree_unsafe
_rmtree_unsafe(fullname, onerror)
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:400: in _rmtree_unsafe
onerror(os.unlink, fullname, sys.exc_info())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
path = 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-4izmxjl32526131575432\\full'
onerror = <function rmtree.<locals>.onerror at 0x0000024C2A4E0DC8>
def _rmtree_unsafe(path, onerror):
try:
with os.scandir(path) as scandir_it:
entries = list(scandir_it)
except OSError:
onerror(os.scandir, path, sys.exc_info())
entries = []
for entry in entries:
fullname = entry.path
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError:
is_dir = False
if is_dir:
try:
if entry.is_symlink():
# This can only happen if someone replaces
# a directory with a symlink after the call to
# os.scandir or entry.is_dir above.
raise OSError("Cannot call rmtree on a symbolic link")
except OSError:
onerror(os.path.islink, fullname, sys.exc_info())
continue
_rmtree_unsafe(fullname, onerror)
else:
try:
> os.unlink(fullname)
E PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py37-win\\tmp\\it-4izmxjl32526131575432\\full\\c24ca9d948-2526133224200-2526136692168-2526131575432'
c:\hostedtoolcache\windows\python\3.7.9\x64\lib\shutil.py:398: PermissionError
{code}
> Python unit tests on windows flaky
> ----------------------------------
>
> Key: BEAM-10921
> URL: https://issues.apache.org/jira/browse/BEAM-10921
> Project: Beam
> Issue Type: Bug
> Components: sdk-py-core
> Reporter: Brian Hulette
> Assignee: Sam Rohde
> Priority: P1
> Labels: flake
> Fix For: Not applicable
>
> Time Spent: 1h 50m
> Remaining Estimate: 0h
>
> Over the past few days python unit tests have been failing frequently. The errors always seem to occur when cleaning up the interactive environment:
> {code}
> ........... [100%]
> ================================== FAILURES ===================================
> _ PipelineInstrumentTest.test_able_to_cache_intermediate_unbounded_source_pcollection _
> [gw2] win32 -- Python 3.5.4 d:\a\beam\beam\sdks\python\target\.tox\py35-win\scripts\python.exe
> self = <apache_beam.runners.interactive.pipeline_instrument_test.PipelineInstrumentTest testMethod=test_able_to_cache_intermediate_unbounded_source_pcollection>
> def setUp(self):
> > ie.new_env()
> apache_beam\runners\interactive\pipeline_instrument_test.py:46:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> apache_beam\runners\interactive\interactive_environment.py:117: in new_env
> _interactive_beam_env.cleanup()
> apache_beam\runners\interactive\interactive_environment.py:273: in cleanup
> cache_manager.cleanup()
> apache_beam\runners\interactive\caching\streaming_cache.py:391: in cleanup
> shutil.rmtree(self._cache_dir)
> c:\hostedtoolcache\windows\python\3.5.4\x64\lib\shutil.py:494: in rmtree
> return _rmtree_unsafe(path, onerror)
> c:\hostedtoolcache\windows\python\3.5.4\x64\lib\shutil.py:384: in _rmtree_unsafe
> _rmtree_unsafe(fullname, onerror)
> c:\hostedtoolcache\windows\python\3.5.4\x64\lib\shutil.py:389: in _rmtree_unsafe
> onerror(os.unlink, fullname, sys.exc_info())
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> path = 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py35-win\\tmp\\it-8vh2z7pi2021914046928\\full'
> onerror = <function rmtree.<locals>.onerror at 0x000001D6C3E5C7B8>
> def _rmtree_unsafe(path, onerror):
> try:
> if os.path.islink(path):
> # symlinks to directories are forbidden, see bug #1669
> raise OSError("Cannot call rmtree on a symbolic link")
> except OSError:
> onerror(os.path.islink, path, sys.exc_info())
> # can't continue even if onerror hook returns
> return
> names = []
> try:
> names = os.listdir(path)
> except OSError:
> onerror(os.listdir, path, sys.exc_info())
> for name in names:
> fullname = os.path.join(path, name)
> try:
> mode = os.lstat(fullname).st_mode
> except OSError:
> mode = 0
> if stat.S_ISDIR(mode):
> _rmtree_unsafe(fullname, onerror)
> else:
> try:
> > os.unlink(fullname)
> E PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'D:\\a\\beam\\beam\\sdks\\python\\target\\.tox\\py35-win\\tmp\\it-8vh2z7pi2021914046928\\full\\ac8879590f-2021876280456-2021876278608-2021914046928'
> c:\hostedtoolcache\windows\python\3.5.4\x64\lib\shutil.py:387: PermissionError
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)