You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:22:52 UTC
[buildstream] 19/22: WIP: mv multiprocessing os-specifics to
platform
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal_seemstowork_20190829
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 00ed4dbb100493fe3ab172b6eff35e7fa6ecef09
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jun 25 15:46:37 2019 +0100
WIP: mv multiprocessing os-specifics to platform
---
src/buildstream/_platform/darwin.py | 12 ++++++++++++
src/buildstream/_platform/platform.py | 4 ++++
src/buildstream/_platform/win32.py | 11 +++++++++--
src/buildstream/_scheduler/jobs/job.py | 5 +++--
src/buildstream/_scheduler/scheduler.py | 4 ----
5 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py
index 86820a3..4efa53a 100644
--- a/src/buildstream/_platform/darwin.py
+++ b/src/buildstream/_platform/darwin.py
@@ -15,6 +15,7 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+import multiprocessing
import os
import resource
@@ -28,6 +29,12 @@ class Darwin(Platform):
# This value comes from OPEN_MAX in syslimits.h
OPEN_MAX = 10240
+ def __init__(self):
+ super().__init__()
+ if None is multiprocessing.get_start_method(allow_none=True):
+ multiprocessing.set_start_method('spawn')
+ self._manager = None
+
def create_sandbox(self, *args, **kwargs):
kwargs['dummy_reason'] = \
"OSXFUSE is not supported and there are no supported sandbox " + \
@@ -62,3 +69,8 @@ class Darwin(Platform):
old_soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
soft_limit = min(max(self.OPEN_MAX, old_soft_limit), hard_limit)
resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
+
+ def make_queue(self):
+ if self._manager is None:
+ self._manager = multiprocessing.Manager()
+ return self._manager.Queue()
diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py
index 42c379e..d0a21a1 100644
--- a/src/buildstream/_platform/platform.py
+++ b/src/buildstream/_platform/platform.py
@@ -17,6 +17,7 @@
# Authors:
# Tristan Maat <tr...@codethink.co.uk>
+import multiprocessing
import os
import platform
import sys
@@ -133,6 +134,9 @@ class Platform():
uname_machine = platform.uname().machine
return Platform.canonicalize_arch(uname_machine)
+ def make_queue(self):
+ return multiprocessing.Queue()
+
##################################################################
# Sandbox functions #
##################################################################
diff --git a/src/buildstream/_platform/win32.py b/src/buildstream/_platform/win32.py
index db70e4a..8844973 100644
--- a/src/buildstream/_platform/win32.py
+++ b/src/buildstream/_platform/win32.py
@@ -1,4 +1,4 @@
-import os
+import multiprocessing
from .._exceptions import PlatformError
from ..sandbox import SandboxNone
@@ -9,8 +9,10 @@ from . import Platform
class Win32(Platform):
def __init__(self):
-
super().__init__()
+ if None is multiprocessing.get_start_method(allow_none=True):
+ multiprocessing.set_start_method('spawn')
+ self._manager = None
def create_sandbox(self, *args, **kwargs):
kwargs['dummy_reason'] = \
@@ -29,3 +31,8 @@ class Win32(Platform):
def maximize_open_file_limit(self):
pass
+
+ def make_queue(self):
+ if self._manager is None:
+ self._manager = multiprocessing.Manager()
+ return self._manager.Queue()
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 9864e4a..f192886 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -33,6 +33,7 @@ import traceback
from ..._exceptions import ImplError, BstError, set_last_task_error, SkipJob
from ..._message import Message, MessageType, unconditional_messages
from ... import _signals, utils
+from ..._platform import Platform
from .jobpickler import pickle_child_job, unpickle_child_job
@@ -184,8 +185,8 @@ class Job():
# Starts the job.
#
def start(self):
-
- self._queue = self._scheduler.manager.Queue()
+ platform = Platform.get_platform()
+ self._queue = platform.make_queue()
self._tries += 1
self._parent_start_listening()
diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py
index 7f876b5..86510c6 100644
--- a/src/buildstream/_scheduler/scheduler.py
+++ b/src/buildstream/_scheduler/scheduler.py
@@ -72,10 +72,6 @@ class Scheduler():
interrupt_callback=None,
ticker_callback=None):
- import multiprocessing
- multiprocessing.set_start_method('spawn')
- self.manager = multiprocessing.Manager()
-
#
# Public members
#