You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:55:14 UTC
[buildstream] branch workspace_list_error_message created (now
936e4ff)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 936e4ff Fixed workspace list error message
This branch includes the following new commits:
new 33cf91c Added `set_resource_limits()` to platform
new bb17975 Added FUSE check, as WSL does not support FUSE yet
new 454b1aa Max-jobs is now defined on a platform basis
new a88b155 Artifact caches are now defined in platform.py
new 82fad9b Adding Dummy sandbox
new 2e2f280 Adding darwin.py (MacOS) platform
new 3a1e27a utils.py: reworked safe_remove
new d0cc954 Sandbox: Ensure that we only import the sandbox subclasses when instantiate a sandbox
new 936e4ff Fixed workspace list error message
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/09: Added FUSE check,
as WSL does not support FUSE yet
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bb17975bebcc5a2faf46fc58f01c14299551711b
Author: knownexus <ph...@codethink.co.uk>
AuthorDate: Thu Aug 30 11:56:23 2018 +0100
Added FUSE check, as WSL does not support FUSE yet
_platform/linux.py: Add FUSE check
And Ensure exception is not raised when performing sandbox availability check
Without this check, WSL would error upon finding FUSE missing
This must be bypassed to allow remote execution
---
buildstream/_platform/linux.py | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index d29a1d5..5543aee 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -17,6 +17,7 @@
# Authors:
# Tristan Maat <tr...@codethink.co.uk>
+import os
import subprocess
from .. import _site
@@ -35,7 +36,11 @@ class Linux(Platform):
super().__init__(context)
self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
- self._user_ns_available = self._check_user_ns_available(context)
+
+ if self._local_sandbox_available():
+ self._user_ns_available = self._check_user_ns_available(context)
+ else:
+ self._user_ns_available = False
self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
@property
@@ -51,6 +56,12 @@ class Linux(Platform):
################################################
# Private Methods #
################################################
+ def _local_sandbox_available(self):
+ try:
+ return os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')
+ except utils.ProgramNotFoundError:
+ return False
+
def _check_user_ns_available(self, context):
# Here, lets check if bwrap is able to create user namespaces,
# issue a warning if it's not available, and save the state
[buildstream] 07/09: utils.py: reworked safe_remove
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3a1e27af54dd47aa54c071a32ceebe58bb21193d
Author: knownexus <ph...@codethink.co.uk>
AuthorDate: Mon Sep 3 16:22:35 2018 +0100
utils.py: reworked safe_remove
non-Linux platforms don't return EISDIR when attempting to unlink a directory
Stopped safe_remove attempting to unlink dirs
Previously safe_remove would attempt to unlink a path
Before attempting to remove it if it was a dir
Now it checks for a dir before that step
---
buildstream/utils.py | 37 ++++++++++++++++++-------------------
1 file changed, 18 insertions(+), 19 deletions(-)
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 60211f3..1e04a31 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -35,6 +35,7 @@ import tempfile
import itertools
import functools
from contextlib import contextmanager
+from stat import S_ISDIR
import psutil
@@ -328,27 +329,25 @@ def safe_remove(path):
Raises:
UtilError: In the case of unexpected system call failures
"""
- if os.path.lexists(path):
-
- # Try to remove anything that is in the way, but issue
- # a warning instead if it removes a non empty directory
- try:
+ try:
+ if S_ISDIR(os.lstat(path).st_mode):
+ os.rmdir(path)
+ else:
os.unlink(path)
- except OSError as e:
- if e.errno != errno.EISDIR:
- raise UtilError("Failed to remove '{}': {}"
- .format(path, e))
-
- try:
- os.rmdir(path)
- except OSError as e:
- if e.errno == errno.ENOTEMPTY:
- return False
- else:
- raise UtilError("Failed to remove '{}': {}"
- .format(path, e))
- return True
+ # File removed/unlinked successfully
+ return True
+
+ except OSError as e:
+ if e.errno == errno.ENOTEMPTY:
+ # Path is non-empty directory
+ return False
+ elif e.errno == errno.ENOENT:
+ # Path does not exist
+ return True
+
+ raise UtilError("Failed to remove '{}': {}"
+ .format(path, e))
def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=False):
[buildstream] 04/09: Artifact caches are now defined in platform.py
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a88b155f1723064036d438678c8ae166153fb635
Author: knownexus <ph...@codethink.co.uk>
AuthorDate: Fri Sep 7 12:12:29 2018 +0100
Artifact caches are now defined in platform.py
This was done so a default exists, but allows platforms to override as needed
_platform/platform.py: Added CAS call function
_platform/linux.py: Added override to CAS call
_platform/unix.py: Remove CAS call
---
buildstream/_platform/linux.py | 11 +++++++----
buildstream/_platform/platform.py | 5 +++++
buildstream/_platform/unix.py | 2 --
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index 5543aee..6a1358a 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -17,7 +17,6 @@
# Authors:
# Tristan Maat <tr...@codethink.co.uk>
-import os
import subprocess
from .. import _site
@@ -33,15 +32,16 @@ class Linux(Platform):
def __init__(self, context):
- super().__init__(context)
-
self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
if self._local_sandbox_available():
self._user_ns_available = self._check_user_ns_available(context)
else:
self._user_ns_available = False
- self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
+
+ # _user_ns_available needs to be set before chaining up to the super class
+ # This is because it will call create_artifact_cache()
+ super().__init__(context)
@property
def artifactcache(self):
@@ -53,6 +53,9 @@ class Linux(Platform):
kwargs['die_with_parent_available'] = self._die_with_parent_available
return SandboxBwrap(*args, **kwargs)
+ def create_artifact_cache(self, context, *, enable_push):
+ return super().create_artifact_cache(context=context, enable_push=self._user_ns_available)
+
################################################
# Private Methods #
################################################
diff --git a/buildstream/_platform/platform.py b/buildstream/_platform/platform.py
index 524985c..bf1e5e8 100644
--- a/buildstream/_platform/platform.py
+++ b/buildstream/_platform/platform.py
@@ -22,6 +22,7 @@ import sys
import resource
from .._exceptions import PlatformError, ImplError
+from .._artifactcache.cascache import CASCache
class Platform():
@@ -39,6 +40,7 @@ class Platform():
def __init__(self, context):
self.context = context
self.set_resource_limits()
+ self._artifact_cache = self.create_artifact_cache(context, enable_push=True)
@classmethod
def create_instance(cls, *args, **kwargs):
@@ -109,3 +111,6 @@ class Platform():
if hard_limit is None:
hard_limit = limits[1]
resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
+
+ def create_artifact_cache(self, context, *, enable_push=True):
+ return CASCache(context=context, enable_push=enable_push)
diff --git a/buildstream/_platform/unix.py b/buildstream/_platform/unix.py
index 0306a4a..c79608c 100644
--- a/buildstream/_platform/unix.py
+++ b/buildstream/_platform/unix.py
@@ -19,7 +19,6 @@
import os
-from .._artifactcache.cascache import CASCache
from .._exceptions import PlatformError
from ..sandbox import SandboxChroot
@@ -31,7 +30,6 @@ class Unix(Platform):
def __init__(self, context):
super().__init__(context)
- self._artifact_cache = CASCache(context)
# Not necessarily 100% reliable, but we want to fail early.
if os.geteuid() != 0:
[buildstream] 05/09: Adding Dummy sandbox
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 82fad9b494a9fe2df9b2bb5b5ef292f838f05664
Author: knownexus <ph...@codethink.co.uk>
AuthorDate: Tue Sep 18 10:36:42 2018 +0100
Adding Dummy sandbox
This is to allow platforms that do not support sandboxing
To error is a controlled/known way
---
buildstream/_platform/linux.py | 13 +++++++-----
buildstream/sandbox/__init__.py | 1 +
buildstream/sandbox/_sandboxdummy.py | 40 ++++++++++++++++++++++++++++++++++++
3 files changed, 49 insertions(+), 5 deletions(-)
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index 6a1358a..d3e0b3e 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -23,7 +23,7 @@ from .. import _site
from .. import utils
from .._artifactcache.cascache import CASCache
from .._message import Message, MessageType
-from ..sandbox import SandboxBwrap
+from ..sandbox import SandboxBwrap, SandboxDummy
from . import Platform
@@ -48,10 +48,13 @@ class Linux(Platform):
return self._artifact_cache
def create_sandbox(self, *args, **kwargs):
- # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
- kwargs['user_ns_available'] = self._user_ns_available
- kwargs['die_with_parent_available'] = self._die_with_parent_available
- return SandboxBwrap(*args, **kwargs)
+ if not self._local_sandbox_available():
+ return SandboxDummy(*args, **kwargs)
+ else:
+ # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
+ kwargs['user_ns_available'] = self._user_ns_available
+ kwargs['die_with_parent_available'] = self._die_with_parent_available
+ return SandboxBwrap(*args, **kwargs)
def create_artifact_cache(self, context, *, enable_push):
return super().create_artifact_cache(context=context, enable_push=self._user_ns_available)
diff --git a/buildstream/sandbox/__init__.py b/buildstream/sandbox/__init__.py
index 2c76e9e..ff40264 100644
--- a/buildstream/sandbox/__init__.py
+++ b/buildstream/sandbox/__init__.py
@@ -21,3 +21,4 @@ from .sandbox import Sandbox, SandboxFlags
from ._sandboxchroot import SandboxChroot
from ._sandboxbwrap import SandboxBwrap
from ._sandboxremote import SandboxRemote
+from ._sandboxdummy import SandboxDummy
diff --git a/buildstream/sandbox/_sandboxdummy.py b/buildstream/sandbox/_sandboxdummy.py
new file mode 100644
index 0000000..51239a4
--- /dev/null
+++ b/buildstream/sandbox/_sandboxdummy.py
@@ -0,0 +1,40 @@
+#
+# Copyright (C) 2017 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+
+from .._exceptions import SandboxError
+from . import Sandbox
+
+
+class SandboxDummy(Sandbox):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ def run(self, command, flags, *, cwd=None, env=None):
+
+ # Fallback to the sandbox default settings for
+ # the cwd and env.
+ #
+ cwd = self._get_work_directory(cwd=cwd)
+ env = self._get_environment(cwd=cwd, env=env)
+
+ if not self._has_command(command[0], env):
+ raise SandboxError("Staged artifacts do not provide command "
+ "'{}'".format(command[0]),
+ reason='missing-command')
+
+ raise SandboxError("This platform does not support local builds")
[buildstream] 09/09: Fixed workspace list error message
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 936e4ffc333a3aa93b369bde1ef0a43fb5af4b86
Author: Phillip Smyth <ph...@codethink.co.uk>
AuthorDate: Mon Sep 24 12:06:32 2018 +0100
Fixed workspace list error message
If no workspaces were found, bst workspace would return `[]`
This has been changed to a nicer message
---
buildstream/_stream.py | 3 +++
tests/frontend/cross_junction_workspace.py | 14 ++++++++------
2 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index f4661cc..70392f5 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -641,6 +641,9 @@ class Stream():
}
workspaces.append(workspace_detail)
+ if not workspaces:
+ workspaces = "No workspaces found"
+
_yaml.dump({
'workspaces': workspaces
})
diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py
index eb2bc2e..bd6823c 100644
--- a/tests/frontend/cross_junction_workspace.py
+++ b/tests/frontend/cross_junction_workspace.py
@@ -93,9 +93,10 @@ def test_close_cross_junction(cli, tmpdir):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert isinstance(loaded.get('workspaces'), list)
- workspaces = loaded['workspaces']
- assert len(workspaces) == 0
+ if not loaded['workspaces'] == "No workspaces found":
+ assert isinstance(loaded.get('workspaces'), list)
+ workspaces = loaded['workspaces']
+ assert len(workspaces) == 0
def test_close_all_cross_junction(cli, tmpdir):
@@ -112,6 +113,7 @@ def test_close_all_cross_junction(cli, tmpdir):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert isinstance(loaded.get('workspaces'), list)
- workspaces = loaded['workspaces']
- assert len(workspaces) == 0
+ if not loaded['workspaces'] == "No workspaces found":
+ assert isinstance(loaded.get('workspaces'), list)
+ workspaces = loaded['workspaces']
+ assert len(workspaces) == 0
[buildstream] 08/09: Sandbox: Ensure that we only import the
sandbox subclasses when instantiate a sandbox
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d0cc95485d1d99ea18da58c185f470932e03fa07
Author: James Ennis <je...@bloomberg.net>
AuthorDate: Fri Sep 21 06:51:03 2018 -0400
Sandbox: Ensure that we only import the sandbox subclasses when instantiate a sandbox
- This patch was required due to not being able to execute `bst` on
a Linux machine without Fuse or Bubblewrap available
---
buildstream/_platform/darwin.py | 2 +-
buildstream/_platform/linux.py | 3 ++-
buildstream/_platform/unix.py | 2 +-
buildstream/sandbox/__init__.py | 2 --
4 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/buildstream/_platform/darwin.py b/buildstream/_platform/darwin.py
index 89e022e..fc9b5d4 100644
--- a/buildstream/_platform/darwin.py
+++ b/buildstream/_platform/darwin.py
@@ -19,7 +19,7 @@ import os
import resource
from .._exceptions import PlatformError
-from ..sandbox import SandboxChroot, SandboxDummy
+from ..sandbox import SandboxDummy
from . import Platform
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index d3e0b3e..5cef2d7 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -23,7 +23,7 @@ from .. import _site
from .. import utils
from .._artifactcache.cascache import CASCache
from .._message import Message, MessageType
-from ..sandbox import SandboxBwrap, SandboxDummy
+from ..sandbox import SandboxDummy
from . import Platform
@@ -51,6 +51,7 @@ class Linux(Platform):
if not self._local_sandbox_available():
return SandboxDummy(*args, **kwargs)
else:
+ from ..sandbox._sandboxbwrap import SandboxBwrap
# Inform the bubblewrap sandbox as to whether it can use user namespaces or not
kwargs['user_ns_available'] = self._user_ns_available
kwargs['die_with_parent_available'] = self._die_with_parent_available
diff --git a/buildstream/_platform/unix.py b/buildstream/_platform/unix.py
index c79608c..6042057 100644
--- a/buildstream/_platform/unix.py
+++ b/buildstream/_platform/unix.py
@@ -20,7 +20,6 @@
import os
from .._exceptions import PlatformError
-from ..sandbox import SandboxChroot
from . import Platform
@@ -40,4 +39,5 @@ class Unix(Platform):
return self._artifact_cache
def create_sandbox(self, *args, **kwargs):
+ from ..sandbox._sandboxchroot import SandboxChroot
return SandboxChroot(*args, **kwargs)
diff --git a/buildstream/sandbox/__init__.py b/buildstream/sandbox/__init__.py
index ff40264..5999aba 100644
--- a/buildstream/sandbox/__init__.py
+++ b/buildstream/sandbox/__init__.py
@@ -18,7 +18,5 @@
# Tristan Maat <tr...@codethink.co.uk>
from .sandbox import Sandbox, SandboxFlags
-from ._sandboxchroot import SandboxChroot
-from ._sandboxbwrap import SandboxBwrap
from ._sandboxremote import SandboxRemote
from ._sandboxdummy import SandboxDummy
[buildstream] 06/09: Adding darwin.py (MacOS) platform
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2e2f280998506c69ba8ebafe16960d8f6e0aa21c
Author: knownexus <ph...@codethink.co.uk>
AuthorDate: Thu Aug 30 12:06:39 2018 +0100
Adding darwin.py (MacOS) platform
Adding functionality to recognise Darwin as a platform in plaform.py
---
buildstream/_platform/darwin.py | 50 +++++++++++++++++++++++++++++++++++++++
buildstream/_platform/platform.py | 12 ++++++----
2 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/buildstream/_platform/darwin.py b/buildstream/_platform/darwin.py
new file mode 100644
index 0000000..89e022e
--- /dev/null
+++ b/buildstream/_platform/darwin.py
@@ -0,0 +1,50 @@
+#
+# Copyright (C) 2017 Codethink Limited
+# Copyright (C) 2018 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import resource
+
+from .._exceptions import PlatformError
+from ..sandbox import SandboxChroot, SandboxDummy
+
+from . import Platform
+
+
+class Darwin(Platform):
+
+ # This value comes from OPEN_MAX in syslimits.h
+ OPEN_MAX = 10240
+
+ def __init__(self, context):
+
+ super().__init__(context)
+
+ @property
+ def artifactcache(self):
+ return self._artifact_cache
+
+ def create_sandbox(self, *args, **kwargs):
+ return SandboxDummy(*args, **kwargs)
+
+ def get_cpu_count(self, cap=None):
+ if cap < os.cpu_count():
+ return cap
+ else:
+ return os.cpu_count()
+
+ def set_resource_limits(self, soft_limit=OPEN_MAX, hard_limit=None):
+ super().set_resource_limits(soft_limit)
diff --git a/buildstream/_platform/platform.py b/buildstream/_platform/platform.py
index bf1e5e8..ac976cc 100644
--- a/buildstream/_platform/platform.py
+++ b/buildstream/_platform/platform.py
@@ -44,19 +44,23 @@ class Platform():
@classmethod
def create_instance(cls, *args, **kwargs):
- if sys.platform.startswith('linux'):
- backend = 'linux'
- else:
- backend = 'unix'
# Meant for testing purposes and therefore hidden in the
# deepest corners of the source code. Try not to abuse this,
# please?
if os.getenv('BST_FORCE_BACKEND'):
backend = os.getenv('BST_FORCE_BACKEND')
+ elif sys.platform.startswith('linux'):
+ backend = 'linux'
+ elif sys.platform.startswith('darwin'):
+ backend = 'darwin'
+ else:
+ backend = 'unix'
if backend == 'linux':
from .linux import Linux as PlatformImpl
+ elif backend == 'darwin':
+ from .darwin import Darwin as PlatformImpl
elif backend == 'unix':
from .unix import Unix as PlatformImpl
else:
[buildstream] 03/09: Max-jobs is now defined on a platform basis
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 454b1aaffd1b9d29a5b772ddb19ea917a612f261
Author: knownexus <ph...@codethink.co.uk>
AuthorDate: Fri Sep 7 12:03:13 2018 +0100
Max-jobs is now defined on a platform basis
This is due to Darwin (MacOS) having a different Max-Jobs limit
_platform/platform.py: Adding max-jobs call
_project.py: redirected 'max-jobs' to point at the platform
---
buildstream/_platform/platform.py | 3 +++
buildstream/_project.py | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/buildstream/_platform/platform.py b/buildstream/_platform/platform.py
index a10c66d..524985c 100644
--- a/buildstream/_platform/platform.py
+++ b/buildstream/_platform/platform.py
@@ -68,6 +68,9 @@ class Platform():
raise PlatformError("Platform needs to be initialized first")
return cls._instance
+ def get_cpu_count(self, cap=None):
+ return min(len(os.sched_getaffinity(0)), cap)
+
##################################################################
# Platform properties #
##################################################################
diff --git a/buildstream/_project.py b/buildstream/_project.py
index b72318a..44e5171 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -38,6 +38,7 @@ from ._loader import Loader
from .element import Element
from ._message import Message, MessageType
from ._includes import Includes
+from ._platform import Platform
# Project Configuration file
@@ -617,7 +618,8 @@ class Project():
# Based on some testing (mainly on AWS), maximum effective
# max-jobs value seems to be around 8-10 if we have enough cores
# users should set values based on workload and build infrastructure
- output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
+ platform = Platform.get_platform()
+ output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
# Export options into variables, if that was requested
output.options.export_variables(output.base_variables)
[buildstream] 01/09: Added `set_resource_limits()` to platform
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch workspace_list_error_message
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 33cf91ceac15964366a3f8b97ff81ed53cc4c2a3
Author: knownexus <ph...@codethink.co.uk>
AuthorDate: Thu Aug 30 11:45:53 2018 +0100
Added `set_resource_limits()` to platform
This has been moved from app.py
As it will have different functionality depending on platform
Once the Darwin (MacOS) platform is added
Removed `resource.setrlimit()` functionality from app.py
Added `resource.setrlimit()` functionality to platform.py as function
---
buildstream/_frontend/app.py | 8 --------
buildstream/_platform/linux.py | 1 -
buildstream/_platform/platform.py | 14 ++++++++++++++
3 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index ccdbb2d..44b4cfb 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -115,14 +115,6 @@ class App():
else:
self.colors = False
- # Increase the soft limit for open file descriptors to the maximum.
- # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
- # Avoid hitting the limit too quickly.
- limits = resource.getrlimit(resource.RLIMIT_NOFILE)
- if limits[0] != limits[1]:
- # Set soft limit to hard limit
- resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
-
# create()
#
# Should be used instead of the regular constructor.
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index a5fd0d6..d29a1d5 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -52,7 +52,6 @@ class Linux(Platform):
# Private Methods #
################################################
def _check_user_ns_available(self, context):
-
# Here, lets check if bwrap is able to create user namespaces,
# issue a warning if it's not available, and save the state
# locally so that we can inform the sandbox to not try it
diff --git a/buildstream/_platform/platform.py b/buildstream/_platform/platform.py
index 8a074eb..a10c66d 100644
--- a/buildstream/_platform/platform.py
+++ b/buildstream/_platform/platform.py
@@ -19,6 +19,7 @@
import os
import sys
+import resource
from .._exceptions import PlatformError, ImplError
@@ -37,6 +38,7 @@ class Platform():
#
def __init__(self, context):
self.context = context
+ self.set_resource_limits()
@classmethod
def create_instance(cls, *args, **kwargs):
@@ -92,3 +94,15 @@ class Platform():
def create_sandbox(self, *args, **kwargs):
raise ImplError("Platform {platform} does not implement create_sandbox()"
.format(platform=type(self).__name__))
+
+ def set_resource_limits(self, soft_limit=None, hard_limit=None):
+ # Need to set resources for _frontend/app.py as this is dependent on the platform
+ # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
+ # Avoid hitting the limit too quickly.
+ limits = resource.getrlimit(resource.RLIMIT_NOFILE)
+ if limits[0] != limits[1]:
+ if soft_limit is None:
+ soft_limit = limits[1]
+ if hard_limit is None:
+ hard_limit = limits[1]
+ resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))