You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:20:55 UTC
[buildstream] 18/19: Apply AsyncioSafeProcess to Stream's
multiprocess
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch tpollard/temp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0089029945246b644b3113827db02fae8943d79f
Author: Tom Pollard <to...@codethink.co.uk>
AuthorDate: Tue Nov 12 16:55:12 2019 +0000
Apply AsyncioSafeProcess to Stream's multiprocess
Note this stops explictly using the get_context object from
multiprocessing which allows for fork to be used in a process
where spawn is the default. This obviously breaks the linux CI
targets for FORCE SPAWN.
---
src/buildstream/{_scheduler => }/_multiprocessing.py | 0
src/buildstream/_scheduler/jobs/job.py | 3 +--
src/buildstream/_stream.py | 6 +++---
3 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/src/buildstream/_scheduler/_multiprocessing.py b/src/buildstream/_multiprocessing.py
similarity index 100%
rename from src/buildstream/_scheduler/_multiprocessing.py
rename to src/buildstream/_multiprocessing.py
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 4cb80b8..8e90997 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -32,8 +32,7 @@ import traceback
from ..._exceptions import ImplError, BstError, SkipJob
from ..._message import Message, MessageType, unconditional_messages
from ...types import FastEnum
-from ... import _signals, utils
-from .. import _multiprocessing
+from ... import _signals, utils, _multiprocessing
from .jobpickler import pickle_child_job, do_pickled_child_job
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 653f150..e62c255 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -63,7 +63,7 @@ from ._profile import Topics, PROFILER
from ._state import State
from .types import _KeyStrength, _SchedulerErrorAction
from .plugin import Plugin
-from . import utils, _yaml, _site, _signals
+from . import utils, _yaml, _site, _signals, _multiprocessing
from . import Scope, Consistency
@@ -151,7 +151,7 @@ class Stream:
def run_in_subprocess(self, func, *args, **kwargs):
assert not self._subprocess
- mp_context = mp.get_context(method="fork")
+ # mp_context = _multiprocessing.get_context(method='fork')
process_name = "stream-{}".format(func.__name__)
self._notify_front_queue = mp.Queue()
@@ -164,7 +164,7 @@ class Stream:
args.insert(0, self._notify_front_queue)
args.insert(0, func)
- self._subprocess = mp_context.Process(
+ self._subprocess = _multiprocessing.AsyncioSafeProcess(
target=Stream._subprocess_main, args=args, kwargs=kwargs, name=process_name
)