You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:39:14 UTC
[buildstream] branch aevri/enable_spawn_ci_7 created (now c2a5141)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at c2a5141 WIP: _fuse/mount: make _run_fuse protected, so subprocess can load
This branch includes the following new commits:
new 0764002 _plugincontext: stable ids for better pickling
new 8004c61 tests/.../missing_dependencies: include git
new 72a9f5a tests/sources: server.start() before using port
new e33eb46 tests/.../ftp_server: be spawn friendly
new 099a637 tests/.../http_server: be spawn friendly
new b48c4cb .gitlab-ci: tests-spawn, enable non-integration
new 9e8c50d .gitlab-ci: tests-spawn, enable all tests
new 6661fd5 testutils/artifactshare: don't hang on error
new 12895b6 tests/artifactshare: safer cleanup_on_sigterm use
new 2d8f542 _fuse/mount private mount() and unmount()
new 8becbcf tests: refactor, extract symlink_host_tools_to_dir
new b01efd5 tests: include 'git' into isolated bin dirs
new e29746c WIP: sandbox bwrap state
new aeb17c6 WIP: _fuse/mount: don't pickle self.__logfile
new c2a5141 WIP: _fuse/mount: make _run_fuse protected, so subprocess can load
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 09/15: tests/artifactshare: safer cleanup_on_sigterm
use
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 12895b62951c3c76678b0b4a21e3ca13568884e0
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Mon Oct 28 15:44:51 2019 +0000
tests/artifactshare: safer cleanup_on_sigterm use
Use the documented path [1] to `pytest_cov.embed.cleanup_on_sigterm()`,
to avoid crashing on some versions.
It turns out that pytest_cov v2.6.1 on my machine doesn't like the way
that we were accessing cleanup_on_sigterm(). Access it in such a way
that we will either get the function or an ImportError, as per the
documentation.
[1]: https://pytest-cov.readthedocs.io/en/latest/subprocess-support.html
---
tests/testutils/artifactshare.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 7a4eda1..e49531d 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -162,11 +162,11 @@ def _start_artifact_server(queue, repodir, quota, index_only):
signal.signal(signal.SIGTERM, lambda signalnum, frame: sys.exit(0))
try:
- import pytest_cov
+ from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
- pytest_cov.embed.cleanup_on_sigterm()
+ cleanup_on_sigterm()
server = stack.enter_context(
create_server(
[buildstream] 04/15: tests/.../ftp_server: be spawn friendly
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e33eb463c9f5b25b1b0267d0694d529f67c6cba5
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 25 15:20:26 2019 +0100
tests/.../ftp_server: be spawn friendly
Make SimpleFtpServer work when using the 'spawn' method of
multiprocessing.
Instead of trying to pickle the FTPServer and friends to the new
process, buffer any config calls and then start the server only in the
new process.
---
tests/testutils/ftp_server.py | 74 ++++++++++++++++++++++++++++++++++---------
1 file changed, 59 insertions(+), 15 deletions(-)
diff --git a/tests/testutils/ftp_server.py b/tests/testutils/ftp_server.py
index 52c05f8..dbe3468 100644
--- a/tests/testutils/ftp_server.py
+++ b/tests/testutils/ftp_server.py
@@ -5,28 +5,72 @@ from pyftpdlib.handlers import FTPHandler
from pyftpdlib.servers import FTPServer
-class SimpleFtpServer(multiprocessing.Process):
+class SimpleFtpServer():
+ # pylint: disable=attribute-defined-outside-init
+
def __init__(self):
- super().__init__()
- self.authorizer = DummyAuthorizer()
- handler = FTPHandler
- handler.authorizer = self.authorizer
- self.server = FTPServer(('127.0.0.1', 0), handler)
+ self._reset()
+
+ def _reset(self):
+ self._process = None
+ self._port = None
+ self._anonymous_dirs = []
+ self._user_list = []
+
+ def start(self):
+ assert self._process is None, "Server already running."
+ queue = multiprocessing.SimpleQueue()
- def run(self):
- self.server.serve_forever()
+ self._process = multiprocessing.Process(
+ target=_run_server,
+ args=(queue, self._anonymous_dirs, self._user_list),
+ )
+
+ self._process.start()
+ self._port = queue.get()
def stop(self):
- self.server.close_all()
- self.server.close()
- self.terminate()
- self.join()
+ assert self._process is not None, "Server not running."
+
+ # Note that when spawning, terminating in this way will cause us to
+ # leak semaphores. This will lead to a warning from Python's semaphore
+ # tracker, which will clean them up for us.
+ #
+ # We could prevent this warning by using alternative methods that would
+ # let the _run_server() function finish, the extra complication doesn't
+ # seem worth it for this test class.
+ #
+ self._process.terminate()
+
+ self._process.join()
+ self._reset()
def allow_anonymous(self, cwd):
- self.authorizer.add_anonymous(cwd)
+ assert self._process is None, "Can't modify server after start()."
+ self._anonymous_dirs.append(cwd)
def add_user(self, user, password, cwd):
- self.authorizer.add_user(user, password, cwd, perm='elradfmwMT')
+ assert self._process is None, "Can't modify server after start()."
+ self._user_list.append((user, password, cwd))
def base_url(self):
- return 'ftp://127.0.0.1:{}'.format(self.server.address[1])
+ assert self._port is not None
+ return 'ftp://127.0.0.1:{}'.format(self._port)
+
+
+def _run_server(queue, anonymous_dirs, user_list):
+ authorizer = DummyAuthorizer()
+ handler = FTPHandler
+ handler.authorizer = authorizer
+
+ for cwd in anonymous_dirs:
+ authorizer.add_anonymous(cwd)
+
+ for user, password, cwd in user_list:
+ authorizer.add_user(user, password, cwd, perm='elradfmwMT')
+
+ server = FTPServer(('127.0.0.1', 0), handler)
+
+ port = server.address[1]
+ queue.put(port)
+ server.serve_forever(handle_exit=True)
[buildstream] 11/15: tests: refactor,
extract symlink_host_tools_to_dir
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8becbcf111a0a97f78645ad94ac0ffc9edd9e582
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 29 17:14:39 2019 +0000
tests: refactor, extract symlink_host_tools_to_dir
Extract a helper function to simplify the repeated pattern of creating a
temporary dir of symlinks for isolating PATH to. Use it in more places.
---
tests/integration/cachedfail.py | 11 +++++-----
tests/sandboxes/missing_dependencies.py | 22 +++++++------------
tests/sandboxes/selection.py | 11 +++++-----
tests/testutils/__init__.py | 1 +
tests/testutils/host_tools.py | 39 +++++++++++++++++++++++++++++++++
5 files changed, 59 insertions(+), 25 deletions(-)
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index 63ad2d4..f624c48 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -20,12 +20,12 @@
import os
import pytest
-from buildstream import utils, _yaml
+from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
from buildstream.testing import cli_integration as cli # pylint: disable=unused-import
from buildstream.testing._utils.site import HAVE_SANDBOX
-from tests.testutils import create_artifact_share
+from tests.testutils import create_artifact_share, symlink_host_tools_to_dir
pytestmark = pytest.mark.integration
@@ -229,9 +229,8 @@ def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error):
@pytest.mark.datafiles(DATA_DIR)
def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
- buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
- buildbox_casd.parent.mkdir()
- os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+ bin_dir = str(tmp_path / 'bin')
+ symlink_host_tools_to_dir(['buildbox-casd'], bin_dir)
project = str(datafiles)
element_path = os.path.join(project, 'elements', 'element.bst')
@@ -257,7 +256,7 @@ def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path):
result1 = cli.run(
project=project,
args=['build', 'element.bst'],
- env={'PATH': str(tmp_path.joinpath('bin')),
+ env={'PATH': bin_dir,
'BST_FORCE_SANDBOX': None})
result1.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
assert cli.get_element_state(project, 'element.bst') == 'buildable'
diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py
index a5bf31e..8935dd8 100644
--- a/tests/sandboxes/missing_dependencies.py
+++ b/tests/sandboxes/missing_dependencies.py
@@ -5,11 +5,12 @@ import os
import pytest
-from buildstream import utils, _yaml
+from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
from buildstream.testing._utils.site import IS_LINUX
from buildstream.testing import cli # pylint: disable=unused-import
+from tests.testutils import symlink_host_tools_to_dir
# Project directory
DATA_DIR = os.path.join(
@@ -18,19 +19,12 @@ DATA_DIR = os.path.join(
)
-def _symlink_host_tools_to_dir(host_tools, dir_):
- dir_.mkdir(exist_ok=True)
- for tool in host_tools:
- target_path = dir_ / tool
- os.symlink(utils.get_host_tool(tool), str(target_path))
-
-
@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
@pytest.mark.datafiles(DATA_DIR)
def test_missing_bwrap_has_nice_error_message(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd and git to work with custom PATH
- bin_dir = tmp_path / "bin"
- _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
+ bin_dir = str(tmp_path / "bin")
+ symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
project = str(datafiles)
element_path = os.path.join(project, 'elements', 'element.bst')
@@ -56,7 +50,7 @@ def test_missing_bwrap_has_nice_error_message(cli, datafiles, tmp_path):
result = cli.run(
project=project,
args=['build', 'element.bst'],
- env={'PATH': str(bin_dir),
+ env={'PATH': bin_dir,
'BST_FORCE_SANDBOX': None})
result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
assert "not found" in result.stderr
@@ -76,8 +70,8 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
bwrap.chmod(0o755)
# Create symlink to buildbox-casd and git to work with custom PATH
- bin_dir = tmp_path / "bin"
- _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
+ bin_dir = str(tmp_path / "bin")
+ symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
project = str(datafiles)
element_path = os.path.join(project, 'elements', 'element3.bst')
@@ -103,7 +97,7 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
result = cli.run(
project=project,
args=['--debug', '--verbose', 'build', 'element3.bst'],
- env={'PATH': str(bin_dir),
+ env={'PATH': bin_dir,
'BST_FORCE_SANDBOX': None})
result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
assert "too old" in result.stderr
diff --git a/tests/sandboxes/selection.py b/tests/sandboxes/selection.py
index b4bbb1b..db937d1 100644
--- a/tests/sandboxes/selection.py
+++ b/tests/sandboxes/selection.py
@@ -19,10 +19,12 @@
import os
import pytest
-from buildstream import utils, _yaml
+from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
from buildstream.testing import cli # pylint: disable=unused-import
+from tests.testutils import symlink_host_tools_to_dir
+
pytestmark = pytest.mark.integration
@@ -66,9 +68,8 @@ def test_force_sandbox(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_dummy_sandbox_fallback(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
- buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
- buildbox_casd.parent.mkdir()
- os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+ bin_dir = str(tmp_path / 'bin')
+ symlink_host_tools_to_dir(['buildbox-casd'], bin_dir)
project = str(datafiles)
element_path = os.path.join(project, 'elements', 'element.bst')
@@ -94,7 +95,7 @@ def test_dummy_sandbox_fallback(cli, datafiles, tmp_path):
result = cli.run(
project=project,
args=['build', 'element.bst'],
- env={'PATH': str(tmp_path.joinpath('bin')),
+ env={'PATH': bin_dir,
'BST_FORCE_SANDBOX': None})
# But if we dont spesify a sandbox then we fall back to dummy, we still
# fail early but only once we know we need a facny sandbox and that
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index 9642ddf..1105803 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -27,6 +27,7 @@ from .artifactshare import create_artifact_share, create_split_share, assert_sha
from .context import dummy_context
from .element_generators import create_element_size, update_element_size
from .junction import generate_junction
+from .host_tools import symlink_host_tools_to_dir
from .runner_integration import wait_for_cache_granularity
from .python_repo import setup_pypi_repo
from .platform import override_platform_uname
diff --git a/tests/testutils/host_tools.py b/tests/testutils/host_tools.py
new file mode 100644
index 0000000..d6c1794
--- /dev/null
+++ b/tests/testutils/host_tools.py
@@ -0,0 +1,39 @@
+#
+# Copyright (C) 2019 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from buildstream import utils
+
+
+# symlink_host_tools_to_dir()
+#
+# Ensure the specified tools are symlinked into the supplied directory.
+#
+# Create # the directory if it doesn't exist.
+#
+# This is useful for isolating tests, such that only the specified tools are
+# available.
+#
+# Args:
+# host_tools (List[str]): The string names of the tools, e.g. ['git', 'bzr'].
+# dir_ (path-like): The path to put the symlinks into.
+#
+def symlink_host_tools_to_dir(host_tools, dir_):
+ os.makedirs(dir_, exist_ok=True)
+ for tool in host_tools:
+ target_path = os.path.join(dir_, tool)
+ os.symlink(utils.get_host_tool(tool), str(target_path))
[buildstream] 08/15: testutils/artifactshare: don't hang on error
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6661fd56bd5a5758c9bf1c1ddf5dbc24f3fbfcac
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Mon Oct 28 15:39:54 2019 +0000
testutils/artifactshare: don't hang on error
Remove a couple of cases where it's possible to make the main test
process hang, waiting for something to appear on a queue.
Make it clearer what is available to the server process by
splitting it out into a non-member function.
Raise a friendlier exception, earlier, if there was a problem starting
the server process.
---
tests/testutils/artifactshare.py | 84 +++++++++++++++++++++-------------------
1 file changed, 45 insertions(+), 39 deletions(-)
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 87b0808..7a4eda1 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -4,7 +4,7 @@ import signal
import sys
from collections import namedtuple
-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
from multiprocessing import Process, Queue
from buildstream._cas import CASCache
@@ -50,50 +50,20 @@ class ArtifactShare():
q = Queue()
- self.process = Process(target=self.run, args=(q,))
+ self.process = Process(
+ target=_start_artifact_server,
+ args=(q, self.repodir, self.quota, self.index_only),
+ )
+
self.process.start()
# Retrieve port from server subprocess
port = q.get()
- self.repo = 'http://localhost:{}'.format(port)
-
- # run():
- #
- # Run the artifact server.
- #
- def run(self, q):
-
- # Handle SIGTERM by calling sys.exit(0), which will raise a SystemExit exception,
- # properly executing cleanup code in `finally` clauses and context managers.
- # This is required to terminate buildbox-casd on SIGTERM.
- signal.signal(signal.SIGTERM, lambda signalnum, frame: sys.exit(0))
-
- try:
- import pytest_cov
- except ImportError:
- pass
- else:
- pytest_cov.embed.cleanup_on_sigterm()
-
- try:
- with create_server(self.repodir,
- quota=self.quota,
- enable_push=True,
- index_only=self.index_only) as server:
- port = server.add_insecure_port('localhost:0')
-
- server.start()
-
- # Send port to parent
- q.put(port)
+ if port is None:
+ raise Exception("Error occurred when starting artifact server.")
- # Sleep until termination by signal
- signal.pause()
-
- except Exception:
- q.put(None)
- raise
+ self.repo = 'http://localhost:{}'.format(port)
# has_object():
#
@@ -183,6 +153,42 @@ class ArtifactShare():
shutil.rmtree(self.directory)
+def _start_artifact_server(queue, repodir, quota, index_only):
+ with ExitStack() as stack:
+ try:
+ # Handle SIGTERM by calling sys.exit(0), which will raise a SystemExit exception,
+ # properly executing cleanup code in `finally` clauses and context managers.
+ # This is required to terminate buildbox-casd on SIGTERM.
+ signal.signal(signal.SIGTERM, lambda signalnum, frame: sys.exit(0))
+
+ try:
+ import pytest_cov
+ except ImportError:
+ pass
+ else:
+ pytest_cov.embed.cleanup_on_sigterm()
+
+ server = stack.enter_context(
+ create_server(
+ repodir,
+ quota=quota,
+ enable_push=True,
+ index_only=index_only,
+ )
+ )
+ port = server.add_insecure_port('localhost:0')
+ server.start()
+ except Exception:
+ queue.put(None)
+ raise
+
+ # Send port to parent
+ queue.put(port)
+
+ # Sleep until termination by signal
+ signal.pause()
+
+
# create_artifact_share()
#
# Create an ArtifactShare for use in a test case
[buildstream] 06/15: .gitlab-ci: tests-spawn, enable non-integration
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b48c4cb01ae6eea13de9957758e43bb2113141be
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Mon Oct 28 09:46:26 2019 +0000
.gitlab-ci: tests-spawn, enable non-integration
Enable all the tests that aren't marked as being integration, as we
ratchet up the required support for spawn mode.
Note that this also doesn't include tests that aren't under the `tests/`
directory.
---
.gitlab-ci.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 21de1f2..f77c485 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -199,7 +199,8 @@ tests-spawn-multiprocessing-start-method:
- mkdir -p "${INTEGRATION_CACHE}"
- useradd -Um buildstream
- chown -R buildstream:buildstream .
- - su buildstream -c "tox -- ${PYTEST_ARGS} tests/{cachekey,elements,format,plugins,internals,sourcecache}"
+ # Note that we're not specifying '--integration' here yet.
+ - su buildstream -c "tox -- --color=yes -n 2 tests/"
# Run type checkers
mypy:
[buildstream] 07/15: .gitlab-ci: tests-spawn, enable all tests
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9e8c50ded00a059ef05cd5c650ec234084c5bf19
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Mon Oct 28 09:50:42 2019 +0000
.gitlab-ci: tests-spawn, enable all tests
Remove the special-case override to restrict which tests are run.
---
.gitlab-ci.yml | 7 -------
1 file changed, 7 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f77c485..bda1be8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -194,13 +194,6 @@ tests-spawn-multiprocessing-start-method:
<<: *tests
variables:
BST_FORCE_START_METHOD: "spawn"
- script:
- # FIXME: Until all the tests pass as normal, override which tests will run here.
- - mkdir -p "${INTEGRATION_CACHE}"
- - useradd -Um buildstream
- - chown -R buildstream:buildstream .
- # Note that we're not specifying '--integration' here yet.
- - su buildstream -c "tox -- --color=yes -n 2 tests/"
# Run type checkers
mypy:
[buildstream] 13/15: WIP: sandbox bwrap state
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e29746c7493b69baa55e082bab55ac18523f1bfe
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 29 10:08:49 2019 +0000
WIP: sandbox bwrap state
This is probably going to need a more substantial refactor - the job
pickler probably shouldn't be reaching into the individual sandboxes
classes like this.
Ideally the state would be stored somewhere more accessible than class
attributes.
---
src/buildstream/_scheduler/jobs/jobpickler.py | 10 ++++++--
src/buildstream/sandbox/_sandboxbwrap.py | 34 +++++++++++++++++++++++++++
2 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_scheduler/jobs/jobpickler.py b/src/buildstream/_scheduler/jobs/jobpickler.py
index b0465ec..ce8e4a6 100644
--- a/src/buildstream/_scheduler/jobs/jobpickler.py
+++ b/src/buildstream/_scheduler/jobs/jobpickler.py
@@ -56,9 +56,11 @@ def pickle_child_job(child_job, projects):
# Note that we need to consider all the state of the program that's
# necessary for the job, this includes e.g. the global state of the node
# module.
+ from ...sandbox._sandboxbwrap import SandboxBwrap
+ sandbox_bwrap_state = SandboxBwrap.save_check_available_status()
node_module_state = node._get_state_for_pickling()
return _pickle_child_job_data(
- (child_job, node_module_state),
+ (child_job, node_module_state, sandbox_bwrap_state),
projects,
)
@@ -77,8 +79,12 @@ def pickle_child_job(child_job, projects):
def do_pickled_child_job(pickled, *child_args):
utils._is_main_process = _not_main_process
- child_job, node_module_state = pickle.load(pickled)
+ child_job, node_module_state, sandbox_bwrap_state = pickle.load(pickled)
node._set_state_from_pickling(node_module_state)
+
+ from ...sandbox._sandboxbwrap import SandboxBwrap
+ SandboxBwrap.load_check_available_status(sandbox_bwrap_state)
+
return child_job.child_action(*child_args)
diff --git a/src/buildstream/sandbox/_sandboxbwrap.py b/src/buildstream/sandbox/_sandboxbwrap.py
index bd60eaf..7911c59 100644
--- a/src/buildstream/sandbox/_sandboxbwrap.py
+++ b/src/buildstream/sandbox/_sandboxbwrap.py
@@ -92,6 +92,40 @@ class SandboxBwrap(Sandbox):
cls.user_ns_available = cls._check_user_ns_available()
+ @classmethod
+ def save_check_available_status(cls):
+ # Note, these ones only get set if bwrap is available:
+ #
+ # cls._uid = os.geteuid()
+ # cls._gid = os.getegid()
+ # cls.user_ns_available = cls._check_user_ns_available()
+ #
+ return (
+ cls._bwrap_exists,
+ cls._die_with_parent_available,
+ cls._dummy_reasons,
+ cls._gid,
+ cls._have_fuse,
+ cls._have_good_bwrap,
+ cls._json_status_available,
+ cls._uid,
+ cls.user_ns_available,
+ )
+
+ @classmethod
+ def load_check_available_status(cls, state):
+ (
+ cls._bwrap_exists,
+ cls._die_with_parent_available,
+ cls._dummy_reasons,
+ cls._gid,
+ cls._have_fuse,
+ cls._have_good_bwrap,
+ cls._json_status_available,
+ cls._uid,
+ cls.user_ns_available,
+ ) = state
+
@staticmethod
def _check_user_ns_available():
# Here, lets check if bwrap is able to create user namespaces,
[buildstream] 14/15: WIP: _fuse/mount: don't pickle self.__logfile
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit aeb17c6cc2cc206e7cabe03e3ae75f2570c024fa
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 29 13:21:36 2019 +0000
WIP: _fuse/mount: don't pickle self.__logfile
---
src/buildstream/_fuse/mount.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/src/buildstream/_fuse/mount.py b/src/buildstream/_fuse/mount.py
index defc41d..9141cdd 100644
--- a/src/buildstream/_fuse/mount.py
+++ b/src/buildstream/_fuse/mount.py
@@ -103,13 +103,19 @@ class Mount():
assert self.__process is None
self.__mountpoint = mountpoint
+
self.__process = Process(target=self.__run_fuse, args=(self.__logfile.name,))
# Ensure the child process does not inherit our signal handlers, if the
# child wants to handle a signal then it will first set its own
# handler, and then unblock it.
with _signals.blocked([signal.SIGTERM, signal.SIGTSTP, signal.SIGINT], ignore=False):
+ # Note that the temporary __logfile isn't picklable, so we must make
+ # sure it is not part of `self` when spawning a process.
+ logfile = self.__logfile
+ self.__logfile = None
self.__process.start()
+ self.__logfile = logfile
while not os.path.ismount(mountpoint):
if not self.__process.is_alive():
[buildstream] 02/15: tests/.../missing_dependencies: include git
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8004c61f6f9a791eb186c3f418df9c76fa5db83a
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Oct 24 16:40:32 2019 +0100
tests/.../missing_dependencies: include git
Symlink `git` into the separate tools bin directory. BuildStream needs
`git` in order to be able to determine its version.
This missing dependency is revealed when you run these tests in `spawn`
mode, as it will re-run the base __init__.__version__ logic in a new
process.
Also take the opportunity to extract _symlink_host_tools_to_dir(), to
re-use some code.
---
tests/sandboxes/missing_dependencies.py | 26 ++++++++++++++++----------
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py
index 975c8eb..a5bf31e 100644
--- a/tests/sandboxes/missing_dependencies.py
+++ b/tests/sandboxes/missing_dependencies.py
@@ -18,13 +18,19 @@ DATA_DIR = os.path.join(
)
+def _symlink_host_tools_to_dir(host_tools, dir_):
+ dir_.mkdir(exist_ok=True)
+ for tool in host_tools:
+ target_path = dir_ / tool
+ os.symlink(utils.get_host_tool(tool), str(target_path))
+
+
@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
@pytest.mark.datafiles(DATA_DIR)
-def test_missing_brwap_has_nice_error_message(cli, datafiles, tmp_path):
- # Create symlink to buildbox-casd to work with custom PATH
- buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
- buildbox_casd.parent.mkdir()
- os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+def test_missing_bwrap_has_nice_error_message(cli, datafiles, tmp_path):
+ # Create symlink to buildbox-casd and git to work with custom PATH
+ bin_dir = tmp_path / "bin"
+ _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
project = str(datafiles)
element_path = os.path.join(project, 'elements', 'element.bst')
@@ -50,7 +56,7 @@ def test_missing_brwap_has_nice_error_message(cli, datafiles, tmp_path):
result = cli.run(
project=project,
args=['build', 'element.bst'],
- env={'PATH': str(tmp_path.joinpath('bin')),
+ env={'PATH': str(bin_dir),
'BST_FORCE_SANDBOX': None})
result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
assert "not found" in result.stderr
@@ -69,9 +75,9 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
bwrap.chmod(0o755)
- # Create symlink to buildbox-casd to work with custom PATH
- buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
- os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+ # Create symlink to buildbox-casd and git to work with custom PATH
+ bin_dir = tmp_path / "bin"
+ _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
project = str(datafiles)
element_path = os.path.join(project, 'elements', 'element3.bst')
@@ -97,7 +103,7 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
result = cli.run(
project=project,
args=['--debug', '--verbose', 'build', 'element3.bst'],
- env={'PATH': str(tmp_path.joinpath('bin')),
+ env={'PATH': str(bin_dir),
'BST_FORCE_SANDBOX': None})
result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
assert "too old" in result.stderr
[buildstream] 12/15: tests: include 'git' into isolated bin dirs
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b01efd5b1a3875237df243cb3ee75b3feba5cb80
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 29 17:17:50 2019 +0000
tests: include 'git' into isolated bin dirs
Symlink `git` into the separate tools bin directory, when trying to
isolate tools. BuildStream needs `git` in order to be able to determine
its version under certain conditions.
This missing dependency is revealed when you run these tests in `spawn`
mode, as it will re-run the base __init__.__version__ logic in a new
process.
---
tests/integration/cachedfail.py | 2 +-
tests/sandboxes/selection.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index f624c48..f26100c 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -230,7 +230,7 @@ def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error):
def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
bin_dir = str(tmp_path / 'bin')
- symlink_host_tools_to_dir(['buildbox-casd'], bin_dir)
+ symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
project = str(datafiles)
element_path = os.path.join(project, 'elements', 'element.bst')
diff --git a/tests/sandboxes/selection.py b/tests/sandboxes/selection.py
index db937d1..ca050b0 100644
--- a/tests/sandboxes/selection.py
+++ b/tests/sandboxes/selection.py
@@ -69,7 +69,7 @@ def test_force_sandbox(cli, datafiles):
def test_dummy_sandbox_fallback(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
bin_dir = str(tmp_path / 'bin')
- symlink_host_tools_to_dir(['buildbox-casd'], bin_dir)
+ symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
project = str(datafiles)
element_path = os.path.join(project, 'elements', 'element.bst')
[buildstream] 03/15: tests/sources: server.start() before using port
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 72a9f5a900ba53b1c000cf8e690f9103e1dba09c
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 25 17:45:07 2019 +0100
tests/sources: server.start() before using port
Re-order the calls of `server.start()` and
`generate_project_file_server()` such that we are not relying on knowing
the server's port before we've called 'start' on it.
This allows us to use spawning or forking to start the server process,
as we won't be relying on being able to share un-picklable things
between the processes.
---
tests/sources/remote.py | 2 +-
tests/sources/tar.py | 9 +++++----
tests/sources/zip.py | 4 ++--
3 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/tests/sources/remote.py b/tests/sources/remote.py
index 5b818b9..9ffa72d 100644
--- a/tests/sources/remote.py
+++ b/tests/sources/remote.py
@@ -204,10 +204,10 @@ def test_use_netrc(cli, datafiles, server_type, tmpdir):
with create_file_server(server_type) as server:
server.add_user('testuser', '12345', project)
- generate_project_file_server(server, project)
server.start()
+ generate_project_file_server(server, project)
result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index fac6f3f..994d568 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -56,6 +56,7 @@ def generate_project(project_dir, tmpdir):
def generate_project_file_server(base_url, project_dir):
+
project_file = os.path.join(project_dir, "project.conf")
_yaml.roundtrip_dump({
'name': 'foo',
@@ -358,13 +359,13 @@ def test_use_netrc(cli, datafiles, server_type, tmpdir):
with create_file_server(server_type) as server:
server.add_user('testuser', '12345', file_server_files)
+ server.start()
+
generate_project_file_server(server.base_url(), project)
src_tar = os.path.join(file_server_files, 'a.tar.gz')
_assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
- server.start()
-
result = cli.run(project=project, args=['source', 'track', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
@@ -398,6 +399,8 @@ def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
with create_file_server(server_type) as server:
server.add_user('otheruser', '12345', file_server_files)
+ server.start()
+
parts = urllib.parse.urlsplit(server.base_url())
base_url = urllib.parse.urlunsplit([parts[0], 'otheruser@{}'.format(parts[1]), *parts[2:]])
generate_project_file_server(base_url, project)
@@ -405,8 +408,6 @@ def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
src_tar = os.path.join(file_server_files, 'a.tar.gz')
_assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
- server.start()
-
result = cli.run(project=project, args=['source', 'track', 'target.bst'])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
diff --git a/tests/sources/zip.py b/tests/sources/zip.py
index 3fd43b4..b298b15 100644
--- a/tests/sources/zip.py
+++ b/tests/sources/zip.py
@@ -212,13 +212,13 @@ def test_use_netrc(cli, datafiles, server_type, tmpdir):
with create_file_server(server_type) as server:
server.add_user('testuser', '12345', file_server_files)
+ server.start()
+
generate_project_file_server(server, project)
src_zip = os.path.join(file_server_files, 'a.zip')
_assemble_zip(os.path.join(str(datafiles), 'content'), src_zip)
- server.start()
-
result = cli.run(project=project, args=['source', 'track', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
[buildstream] 10/15: _fuse/mount private mount() and unmount()
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2d8f542d6ce357527df9ba71cbdbc9c68326f5de
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 29 13:13:41 2019 +0000
_fuse/mount private mount() and unmount()
These aren't used by anything else, so make them private.
---
src/buildstream/_fuse/mount.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/src/buildstream/_fuse/mount.py b/src/buildstream/_fuse/mount.py
index 52d3ed6..defc41d 100644
--- a/src/buildstream/_fuse/mount.py
+++ b/src/buildstream/_fuse/mount.py
@@ -91,14 +91,14 @@ class Mount():
def __init__(self, fuse_mount_options=None):
self._fuse_mount_options = {} if fuse_mount_options is None else fuse_mount_options
- # mount():
+ # __mount():
#
# User facing API for mounting a fuse subclass implementation
#
# Args:
# (str): Location to mount this fuse fs
#
- def mount(self, mountpoint):
+ def __mount(self, mountpoint):
assert self.__process is None
@@ -119,11 +119,11 @@ class Mount():
time.sleep(1 / 100)
- # unmount():
+ # __unmount():
#
# User facing API for unmounting a fuse subclass implementation
#
- def unmount(self):
+ def __unmount(self):
# Terminate child process and join
if self.__process is not None:
@@ -156,12 +156,12 @@ class Mount():
with utils._tempnamedfile() as logfile:
self.__logfile = logfile
- self.mount(mountpoint)
+ self.__mount(mountpoint)
try:
- with _signals.terminator(self.unmount):
+ with _signals.terminator(self.__unmount):
yield
finally:
- self.unmount()
+ self.__unmount()
self.__logfile = None
[buildstream] 15/15: WIP: _fuse/mount: make _run_fuse protected,
so subprocess can load
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c2a5141a78284b6c522f0aa9fb0dd84f635f1f1a
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 29 13:38:22 2019 +0000
WIP: _fuse/mount: make _run_fuse protected, so subprocess can load
---
src/buildstream/_fuse/mount.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_fuse/mount.py b/src/buildstream/_fuse/mount.py
index 9141cdd..ce08512 100644
--- a/src/buildstream/_fuse/mount.py
+++ b/src/buildstream/_fuse/mount.py
@@ -104,7 +104,7 @@ class Mount():
self.__mountpoint = mountpoint
- self.__process = Process(target=self.__run_fuse, args=(self.__logfile.name,))
+ self.__process = Process(target=self._run_fuse, args=(self.__logfile.name,))
# Ensure the child process does not inherit our signal handlers, if the
# child wants to handle a signal then it will first set its own
@@ -188,7 +188,7 @@ class Mount():
################################################
# Child Process #
################################################
- def __run_fuse(self, filename):
+ def _run_fuse(self, filename):
# Override stdout/stderr to the file given as a pointer, that way our parent process can get our output
out = open(filename, "w")
os.dup2(out.fileno(), sys.stdout.fileno())
[buildstream] 01/15: _plugincontext: stable ids for better pickling
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0764002a98b731ccf01537cd92d3b7b74f3377b9
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Oct 24 09:50:06 2019 +0100
_plugincontext: stable ids for better pickling
Use the 'identifier' argument of PluginBase's make_plugin_source(), to
ensure that the plugins have a consistent namespace when pickled and
then unpickled in another process.
This means that we can pickle plugins that are more than an entrypoint,
e.g. ones that have multiple classes in the same file.
This enables the `tests/frontend/mirror.py::test_mirror_fetch_multi`
test to pass, which uses `fetch_source` plugin. That plugin has more
than one class in it's file.
---
src/buildstream/_plugincontext.py | 31 +++++++++++++++++++++++--------
1 file changed, 23 insertions(+), 8 deletions(-)
diff --git a/src/buildstream/_plugincontext.py b/src/buildstream/_plugincontext.py
index 9e32f49..b07c2b3 100644
--- a/src/buildstream/_plugincontext.py
+++ b/src/buildstream/_plugincontext.py
@@ -46,6 +46,12 @@ class PluginContext():
def __init__(self, plugin_base, base_type, site_plugin_path, *,
plugin_origins=None, format_versions={}):
+ # For pickling across processes, make sure this context has a unique
+ # identifier, which we prepend to the identifier of each PluginSource.
+ # This keeps plugins loaded during the first and second pass distinct
+ # from eachother.
+ self._identifier = str(id(self))
+
# The plugin kinds which were loaded
self.loaded_dependencies = []
@@ -59,12 +65,17 @@ class PluginContext():
# The PluginSource object
self._plugin_base = plugin_base
self._site_plugin_path = site_plugin_path
- self._site_source = plugin_base.make_plugin_source(
- searchpath=self._site_plugin_path,
- )
self._alternate_sources = {}
self._format_versions = format_versions
+ self._init_site_source()
+
+ def _init_site_source(self):
+ self._site_source = self._plugin_base.make_plugin_source(
+ searchpath=self._site_plugin_path,
+ identifier=self._identifier + 'site',
+ )
+
def __getstate__(self):
state = self.__dict__.copy()
@@ -98,9 +109,7 @@ class PluginContext():
# of the PluginSource. We would also have to recreate `_types` as it
# was before unpickling them. We are not using this method in
# BuildStream, so the identifier is not restored here.
- self._site_source = self._plugin_base.make_plugin_source(
- searchpath=self._site_plugin_path,
- )
+ self._init_site_source()
# lookup():
#
@@ -126,7 +135,10 @@ class PluginContext():
def _get_local_plugin_source(self, path):
if ('local', path) not in self._alternate_sources:
# key by a tuple to avoid collision
- source = self._plugin_base.make_plugin_source(searchpath=[path])
+ source = self._plugin_base.make_plugin_source(
+ searchpath=[path],
+ identifier=self._identifier + path,
+ )
# Ensure that sources never get garbage collected,
# as they'll take the plugins with them.
self._alternate_sources[('local', path)] = source
@@ -167,7 +179,10 @@ class PluginContext():
# The plugin didn't have an accompanying YAML file
defaults = None
- source = self._plugin_base.make_plugin_source(searchpath=[os.path.dirname(location)])
+ source = self._plugin_base.make_plugin_source(
+ searchpath=[os.path.dirname(location)],
+ identifier=self._identifier + os.path.dirname(location),
+ )
self._alternate_sources[('pip', package_name)] = source
else:
[buildstream] 05/15: tests/.../http_server: be spawn friendly
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/enable_spawn_ci_7
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 099a637aa29770101c15988349785abe096dde26
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 25 15:20:26 2019 +0100
tests/.../http_server: be spawn friendly
Make SimpleHttpServer work when using the 'spawn' method of
multiprocessing.
Instead of trying to pickle the HTTPServer and friends to the new
process, buffer any config calls and then start the server only in the
new process.
---
tests/testutils/http_server.py | 59 +++++++++++++++++++++++++++++++-----------
1 file changed, 44 insertions(+), 15 deletions(-)
diff --git a/tests/testutils/http_server.py b/tests/testutils/http_server.py
index b72e745..57e813f 100644
--- a/tests/testutils/http_server.py
+++ b/tests/testutils/http_server.py
@@ -81,30 +81,59 @@ class AuthHTTPServer(HTTPServer):
super().__init__(*args, **kwargs)
-class SimpleHttpServer(multiprocessing.Process):
+class SimpleHttpServer():
+ # pylint: disable=attribute-defined-outside-init
+
def __init__(self):
- super().__init__()
- self.server = AuthHTTPServer(('127.0.0.1', 0), RequestHandler)
- self.started = False
+ self._reset()
+
+ def _reset(self):
+ self._process = None
+ self._port = None
+ self._anonymous_dir = None
+ self._user_list = []
def start(self):
- self.started = True
- super().start()
+ assert self._process is None, "Server already running."
+ queue = multiprocessing.SimpleQueue()
+
+ self._process = multiprocessing.Process(
+ target=_run_server,
+ args=(queue, self._anonymous_dir, self._user_list),
+ )
- def run(self):
- self.server.serve_forever()
+ self._process.start()
+ self._port = queue.get()
def stop(self):
- if not self.started:
- return
- self.terminate()
- self.join()
+ assert self._process is not None, "Server not running."
+ self._process.terminate()
+ self._process.join()
+ self._reset()
def allow_anonymous(self, cwd):
- self.server.anonymous_dir = cwd
+ assert self._process is None, "Can't modify server after start()."
+ assert self._anonymous_dir is None, "Only one anonymous_dir is supported."
+ self._anonymous_dir = cwd
def add_user(self, user, password, cwd):
- self.server.users[user] = (password, cwd)
+ assert self._process is None, "Can't modify server after start()."
+ self._user_list.append((user, password, cwd))
def base_url(self):
- return 'http://127.0.0.1:{}'.format(self.server.server_port)
+ assert self._port is not None
+ return 'http://127.0.0.1:{}'.format(self._port)
+
+
+def _run_server(queue, anonymous_dir, user_list):
+ server = AuthHTTPServer(('127.0.0.1', 0), RequestHandler)
+
+ if anonymous_dir is not None:
+ server.anonymous_dir = anonymous_dir
+
+ for user, password, cwd in user_list:
+ server.users[user] = (password, cwd)
+
+ queue.put(server.server_port)
+
+ server.serve_forever()