You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:14:28 UTC
[buildstream] branch aevri/enable_spawn_ci_6 created (now 5e5b901)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch aevri/enable_spawn_ci_6
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 5e5b901 .gitlab-ci: tests-spawn, enable non-integration
This branch includes the following new commits:
new 44db0a0 tests/sources: server.start() before using port
new 6fdfed3 tests/.../ftp_server: be spawn friendly
new b0164e0 tests/.../http_server: be spawn friendly
new 5e5b901 .gitlab-ci: tests-spawn, enable non-integration
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/04: tests/.../ftp_server: be spawn friendly
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/enable_spawn_ci_6
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6fdfed345d7b92ac90c7e34b55f16b3dde0d1d11
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 25 15:20:26 2019 +0100
tests/.../ftp_server: be spawn friendly
Make SimpleFtpServer work when using the 'spawn' method of
multiprocessing.
Instead of trying to pickle the FTPServer and friends to the new
process, buffer any config calls and then start the server only in the
new process.
---
tests/testutils/ftp_server.py | 74 ++++++++++++++++++++++++++++++++++---------
1 file changed, 59 insertions(+), 15 deletions(-)
diff --git a/tests/testutils/ftp_server.py b/tests/testutils/ftp_server.py
index 52c05f8..dbe3468 100644
--- a/tests/testutils/ftp_server.py
+++ b/tests/testutils/ftp_server.py
@@ -5,28 +5,72 @@ from pyftpdlib.handlers import FTPHandler
from pyftpdlib.servers import FTPServer
-class SimpleFtpServer(multiprocessing.Process):
+class SimpleFtpServer():
+ # pylint: disable=attribute-defined-outside-init
+
def __init__(self):
- super().__init__()
- self.authorizer = DummyAuthorizer()
- handler = FTPHandler
- handler.authorizer = self.authorizer
- self.server = FTPServer(('127.0.0.1', 0), handler)
+ self._reset()
+
+ def _reset(self):
+ self._process = None
+ self._port = None
+ self._anonymous_dirs = []
+ self._user_list = []
+
+ def start(self):
+ assert self._process is None, "Server already running."
+ queue = multiprocessing.SimpleQueue()
- def run(self):
- self.server.serve_forever()
+ self._process = multiprocessing.Process(
+ target=_run_server,
+ args=(queue, self._anonymous_dirs, self._user_list),
+ )
+
+ self._process.start()
+ self._port = queue.get()
def stop(self):
- self.server.close_all()
- self.server.close()
- self.terminate()
- self.join()
+ assert self._process is not None, "Server not running."
+
+ # Note that when spawning, terminating in this way will cause us to
+ # leak semaphores. This will lead to a warning from Python's semaphore
+ # tracker, which will clean them up for us.
+ #
+ # We could prevent this warning by using alternative methods that would
+ # let the _run_server() function finish, the extra complication doesn't
+ # seem worth it for this test class.
+ #
+ self._process.terminate()
+
+ self._process.join()
+ self._reset()
def allow_anonymous(self, cwd):
- self.authorizer.add_anonymous(cwd)
+ assert self._process is None, "Can't modify server after start()."
+ self._anonymous_dirs.append(cwd)
def add_user(self, user, password, cwd):
- self.authorizer.add_user(user, password, cwd, perm='elradfmwMT')
+ assert self._process is None, "Can't modify server after start()."
+ self._user_list.append((user, password, cwd))
def base_url(self):
- return 'ftp://127.0.0.1:{}'.format(self.server.address[1])
+ assert self._port is not None
+ return 'ftp://127.0.0.1:{}'.format(self._port)
+
+
+def _run_server(queue, anonymous_dirs, user_list):
+ authorizer = DummyAuthorizer()
+ handler = FTPHandler
+ handler.authorizer = authorizer
+
+ for cwd in anonymous_dirs:
+ authorizer.add_anonymous(cwd)
+
+ for user, password, cwd in user_list:
+ authorizer.add_user(user, password, cwd, perm='elradfmwMT')
+
+ server = FTPServer(('127.0.0.1', 0), handler)
+
+ port = server.address[1]
+ queue.put(port)
+ server.serve_forever(handle_exit=True)
[buildstream] 04/04: .gitlab-ci: tests-spawn, enable non-integration
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/enable_spawn_ci_6
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5e5b90196cf5e158c753ff95ecca624fbcea495a
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Mon Oct 28 09:46:26 2019 +0000
.gitlab-ci: tests-spawn, enable non-integration
Enable all the tests that aren't marked as being integration, as we
ratchet up the required support for spawn mode.
Note that this also doesn't include tests that aren't under the `tests/`
directory.
---
.gitlab-ci.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f669e19..f77c485 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -199,7 +199,8 @@ tests-spawn-multiprocessing-start-method:
- mkdir -p "${INTEGRATION_CACHE}"
- useradd -Um buildstream
- chown -R buildstream:buildstream .
- - su buildstream -c "tox -- ${PYTEST_ARGS} tests/{artifactcache,cachekey,elements,format,frontend,internals,plugins,sourcecache}"
+ # Note that we're not specifying '--integration' here yet.
+ - su buildstream -c "tox -- --color=yes -n 2 tests/"
# Run type checkers
mypy:
[buildstream] 01/04: tests/sources: server.start() before using port
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/enable_spawn_ci_6
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 44db0a07b2797c79bb28dfe2c3152dc89746ebd3
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 25 17:45:07 2019 +0100
tests/sources: server.start() before using port
Re-order the calls of `server.start()` and
`generate_project_file_server()` such that we are not relying on knowing
the server's port before we've called 'start' on it.
This allows us to use spawning or forking to start the server process,
as we won't be relying on being able to share un-picklable things
between the processes.
---
tests/sources/remote.py | 2 +-
tests/sources/tar.py | 9 +++++----
tests/sources/zip.py | 4 ++--
3 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/tests/sources/remote.py b/tests/sources/remote.py
index 5b818b9..9ffa72d 100644
--- a/tests/sources/remote.py
+++ b/tests/sources/remote.py
@@ -204,10 +204,10 @@ def test_use_netrc(cli, datafiles, server_type, tmpdir):
with create_file_server(server_type) as server:
server.add_user('testuser', '12345', project)
- generate_project_file_server(server, project)
server.start()
+ generate_project_file_server(server, project)
result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index fac6f3f..994d568 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -56,6 +56,7 @@ def generate_project(project_dir, tmpdir):
def generate_project_file_server(base_url, project_dir):
+
project_file = os.path.join(project_dir, "project.conf")
_yaml.roundtrip_dump({
'name': 'foo',
@@ -358,13 +359,13 @@ def test_use_netrc(cli, datafiles, server_type, tmpdir):
with create_file_server(server_type) as server:
server.add_user('testuser', '12345', file_server_files)
+ server.start()
+
generate_project_file_server(server.base_url(), project)
src_tar = os.path.join(file_server_files, 'a.tar.gz')
_assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
- server.start()
-
result = cli.run(project=project, args=['source', 'track', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
@@ -398,6 +399,8 @@ def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
with create_file_server(server_type) as server:
server.add_user('otheruser', '12345', file_server_files)
+ server.start()
+
parts = urllib.parse.urlsplit(server.base_url())
base_url = urllib.parse.urlunsplit([parts[0], 'otheruser@{}'.format(parts[1]), *parts[2:]])
generate_project_file_server(base_url, project)
@@ -405,8 +408,6 @@ def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
src_tar = os.path.join(file_server_files, 'a.tar.gz')
_assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
- server.start()
-
result = cli.run(project=project, args=['source', 'track', 'target.bst'])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
diff --git a/tests/sources/zip.py b/tests/sources/zip.py
index 3fd43b4..b298b15 100644
--- a/tests/sources/zip.py
+++ b/tests/sources/zip.py
@@ -212,13 +212,13 @@ def test_use_netrc(cli, datafiles, server_type, tmpdir):
with create_file_server(server_type) as server:
server.add_user('testuser', '12345', file_server_files)
+ server.start()
+
generate_project_file_server(server, project)
src_zip = os.path.join(file_server_files, 'a.zip')
_assemble_zip(os.path.join(str(datafiles), 'content'), src_zip)
- server.start()
-
result = cli.run(project=project, args=['source', 'track', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
[buildstream] 03/04: tests/.../http_server: be spawn friendly
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/enable_spawn_ci_6
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b0164e05bb71ac4d62cd62d475230ca14e786d96
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 25 15:20:26 2019 +0100
tests/.../http_server: be spawn friendly
Make SimpleHttpServer work when using the 'spawn' method of
multiprocessing.
Instead of trying to pickle the HTTPServer and friends to the new
process, buffer any config calls and then start the server only in the
new process.
---
tests/testutils/http_server.py | 59 +++++++++++++++++++++++++++++++-----------
1 file changed, 44 insertions(+), 15 deletions(-)
diff --git a/tests/testutils/http_server.py b/tests/testutils/http_server.py
index b72e745..57e813f 100644
--- a/tests/testutils/http_server.py
+++ b/tests/testutils/http_server.py
@@ -81,30 +81,59 @@ class AuthHTTPServer(HTTPServer):
super().__init__(*args, **kwargs)
-class SimpleHttpServer(multiprocessing.Process):
+class SimpleHttpServer():
+ # pylint: disable=attribute-defined-outside-init
+
def __init__(self):
- super().__init__()
- self.server = AuthHTTPServer(('127.0.0.1', 0), RequestHandler)
- self.started = False
+ self._reset()
+
+ def _reset(self):
+ self._process = None
+ self._port = None
+ self._anonymous_dir = None
+ self._user_list = []
def start(self):
- self.started = True
- super().start()
+ assert self._process is None, "Server already running."
+ queue = multiprocessing.SimpleQueue()
+
+ self._process = multiprocessing.Process(
+ target=_run_server,
+ args=(queue, self._anonymous_dir, self._user_list),
+ )
- def run(self):
- self.server.serve_forever()
+ self._process.start()
+ self._port = queue.get()
def stop(self):
- if not self.started:
- return
- self.terminate()
- self.join()
+ assert self._process is not None, "Server not running."
+ self._process.terminate()
+ self._process.join()
+ self._reset()
def allow_anonymous(self, cwd):
- self.server.anonymous_dir = cwd
+ assert self._process is None, "Can't modify server after start()."
+ assert self._anonymous_dir is None, "Only one anonymous_dir is supported."
+ self._anonymous_dir = cwd
def add_user(self, user, password, cwd):
- self.server.users[user] = (password, cwd)
+ assert self._process is None, "Can't modify server after start()."
+ self._user_list.append((user, password, cwd))
def base_url(self):
- return 'http://127.0.0.1:{}'.format(self.server.server_port)
+ assert self._port is not None
+ return 'http://127.0.0.1:{}'.format(self._port)
+
+
+def _run_server(queue, anonymous_dir, user_list):
+ server = AuthHTTPServer(('127.0.0.1', 0), RequestHandler)
+
+ if anonymous_dir is not None:
+ server.anonymous_dir = anonymous_dir
+
+ for user, password, cwd in user_list:
+ server.users[user] = (password, cwd)
+
+ queue.put(server.server_port)
+
+ server.serve_forever()