You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:44:43 UTC
[buildstream] 17/17: Apply AsyncioSafeProcess to Stream's
multiprocess
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tpollard/buildsubprocess
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ac2433491154c458e8d0c7cc68ccd6cded508153
Author: Tom Pollard <to...@codethink.co.uk>
AuthorDate: Tue Nov 12 16:55:12 2019 +0000
Apply AsyncioSafeProcess to Stream's multiprocess
Note this stops explictly using the get_context object from
multiprocessing which allows for fork to be used in a process
where spawn is the default. This obviously breaks the linux CI
targets for FORCE SPAWN.
---
src/buildstream/{_scheduler => }/_multiprocessing.py | 0
src/buildstream/_scheduler/jobs/job.py | 3 +--
src/buildstream/_stream.py | 6 +++---
3 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/src/buildstream/_scheduler/_multiprocessing.py b/src/buildstream/_multiprocessing.py
similarity index 100%
rename from src/buildstream/_scheduler/_multiprocessing.py
rename to src/buildstream/_multiprocessing.py
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 460f63d..9d135c6 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -33,8 +33,7 @@ import traceback
from ..._exceptions import ImplError, BstError, SkipJob
from ..._message import Message, MessageType, unconditional_messages
from ...types import FastEnum
-from ... import _signals, utils
-from .. import _multiprocessing
+from ... import _signals, utils, _multiprocessing
from .jobpickler import pickle_child_job, do_pickled_child_job
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index ebabcac..86eea49 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -65,7 +65,7 @@ from ._profile import Topics, PROFILER
from ._state import State
from .types import _KeyStrength, _SchedulerErrorAction
from .plugin import Plugin
-from . import utils, _yaml, _site, _signals
+from . import utils, _yaml, _site, _signals, _multiprocessing
from . import Scope
@@ -152,7 +152,7 @@ class Stream:
def run_in_subprocess(self, func, *args, **kwargs):
assert not self._subprocess
- mp_context = mp.get_context(method="fork")
+ # mp_context = _multiprocessing.get_context(method='fork')
process_name = "stream-{}".format(func.__name__)
self._notify_front_queue = mp.Queue()
@@ -165,7 +165,7 @@ class Stream:
args.insert(0, self._notify_front_queue)
args.insert(0, func)
- self._subprocess = mp_context.Process(
+ self._subprocess = _multiprocessing.AsyncioSafeProcess(
target=Stream._subprocess_main, args=args, kwargs=kwargs, name=process_name
)