You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:47:02 UTC
[buildstream] branch jmac/virtual_directories_test created (now
a3dfc3f)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at a3dfc3f _stream.py: Alter _checkout_hardlinks to use can_destroy
This branch includes the following new commits:
new 5034957 Revert "Replace bwrap checks with calls to check_bwrap_version"
new c5d1a5a Revert "_site.py: Add check_bwrap_version() function"
new bb16a98 Add BST_VIRTUAL_DIRECTORY flag for element plugins
new 9c43670 Add the virtual directory class 'Directory' and one implementation, _filebaseddirectory.py. These are not used anywhere yet.
new ab8dd49 Sandbox.py: Rename __root to _root to allow its use by subclasses.
new 7010dab Sandbox.py: Add get_virtual_directory and get_virtual_toplevel_directory
new ad87b92 _stream.py: Convert to virtual directories.
new 692d76a element.py: Conversion to use virtual directories
new f9a3b53 plugins/elements/compose.py: Convert to virtual directories
new ca09714 plugins/elements/import.py: Convert to virtual directories
new 379ca4c sandbox/_mount.py, sandbox/_sandboxbwrap.py: Remove instances of get_directory
new b842699 plugins/elements/stack.py: Convert to virtual directories
new c52a592 scriptelement.py: Convert to virtual directories
new fbde34e Directory.py: Change the mtime interface into mark_unmodified/list_modified_paths
new cbc5539 Directory.py: link_ok=>can_link (consistent with other use)
new 2e8748a Implement can_destroy flag in _filebaseddirectory.py
new a3dfc3f _stream.py: Alter _checkout_hardlinks to use can_destroy
The 17 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 15/17: Directory.py: link_ok=>can_link (consistent
with other use)
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cbc55394d2a585a9ca5a3038d2c2e8f25d52882e
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Wed May 9 12:51:13 2018 +0100
Directory.py: link_ok=>can_link (consistent with other use)
---
buildstream/sandbox/Directory.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py
index 4107496..0a60fce 100644
--- a/buildstream/sandbox/Directory.py
+++ b/buildstream/sandbox/Directory.py
@@ -53,7 +53,7 @@ class Directory():
# Import and export of files and links
def import_files(self, external_pathspec: any, files: List[str] = None,
report_written: bool = True, update_utimes: bool = False,
- link_ok: bool = False) -> FileListResult:
+ can_link: bool = False) -> FileListResult:
"""Imports some or all files from external_path into this directory.
Keyword arguments: external_pathspec: Either a string
@@ -80,7 +80,7 @@ class Directory():
raise NotImplementedError()
- def export_files(self, to_directory: str, link_ok: bool = False) -> None:
+ def export_files(self, to_directory: str, can_link: bool = False) -> None:
"""Copies everything from this into to_directory.
Arguments:
[buildstream] 05/17: Sandbox.py: Rename __root to _root to allow
its use by subclasses.
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ab8dd4902f313f0a672185b65d87bbe74594d6a8
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:14:37 2018 +0100
Sandbox.py: Rename __root to _root to allow its use by subclasses.
Since access to get_directories is now blocked for some plugins,
and the subclasses of Sandbox do not have configuration defined
by YAML files, they need another way to get at the root directory.
NB Could this be done just with get_virtual_directory and .external_directory?
---
buildstream/sandbox/_sandboxchroot.py | 2 +-
buildstream/sandbox/sandbox.py | 9 +++++----
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py
index 7f27f50..d03e575 100644
--- a/buildstream/sandbox/_sandboxchroot.py
+++ b/buildstream/sandbox/_sandboxchroot.py
@@ -86,7 +86,7 @@ class SandboxChroot(Sandbox):
# Nonetheless a better solution could perhaps be found.
rootfs = stack.enter_context(utils._tempdir(dir='/var/run/buildstream'))
- stack.enter_context(self.create_devices(self.get_directory(), flags))
+ stack.enter_context(self.create_devices(self._root, flags))
stack.enter_context(self.mount_dirs(rootfs, flags, stdout, stderr))
if flags & SandboxFlags.INTERACTIVE:
diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py
index 4f0bd07..c99bbaf 100644
--- a/buildstream/sandbox/sandbox.py
+++ b/buildstream/sandbox/sandbox.py
@@ -99,11 +99,12 @@ class Sandbox():
self.__stdout = kwargs['stdout']
self.__stderr = kwargs['stderr']
- # Setup the directories
+ # Setup the directories. Root should be available to subclasses, hence
+ # being single-underscore. The others are private to this class.
+ self._root = os.path.join(directory, 'root')
self.__directory = directory
- self.__root = os.path.join(self.__directory, 'root')
self.__scratch = os.path.join(self.__directory, 'scratch')
- for directory_ in [self.__root, self.__scratch]:
+ for directory_ in [self._root, self.__scratch]:
os.makedirs(directory_, exist_ok=True)
def get_directory(self):
@@ -118,7 +119,7 @@ class Sandbox():
"""
if self.__allow_real_directory:
- return self.__root
+ return self._root
else:
raise BstError("You can't use get_directory")
[buildstream] 03/17: Add BST_VIRTUAL_DIRECTORY flag for element
plugins
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bb16a98d610ca1b37bc2d5bfe45f165306e4b130
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Wed May 2 18:24:33 2018 +0100
Add BST_VIRTUAL_DIRECTORY flag for element plugins
---
buildstream/element.py | 10 +++++++++-
buildstream/sandbox/sandbox.py | 13 ++++++++++---
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index c563b79..c3c3513 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -192,6 +192,13 @@ class Element(Plugin):
*Since: 1.2*
"""
+ BST_VIRTUAL_DIRECTORY = False
+ """Whether to raise exceptions if an element uses Sandbox.get_directory
+ instead of Sandbox.get_virtual_directory.
+
+ *Since: 1.2*
+ """
+
def __init__(self, context, project, artifacts, meta, plugin_conf):
super().__init__(meta.name, context, project, meta.provenance, "element")
@@ -2052,7 +2059,8 @@ class Element(Plugin):
directory,
stdout=stdout,
stderr=stderr,
- config=config)
+ config=config,
+ allow_real_directory=not self.BST_VIRTUAL_DIRECTORY)
yield sandbox
else:
diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py
index 3ab75f1..4f0bd07 100644
--- a/buildstream/sandbox/sandbox.py
+++ b/buildstream/sandbox/sandbox.py
@@ -31,7 +31,7 @@ See also: :ref:`sandboxing`.
"""
import os
-from .._exceptions import ImplError
+from .._exceptions import ImplError, BstError
class SandboxFlags():
@@ -92,6 +92,8 @@ class Sandbox():
self.__cwd = None
self.__env = None
self.__mount_sources = {}
+ self.__allow_real_directory = kwargs['allow_real_directory']
+
# Configuration from kwargs common to all subclasses
self.__config = kwargs['config']
self.__stdout = kwargs['stdout']
@@ -108,12 +110,17 @@ class Sandbox():
"""Fetches the sandbox root directory
The root directory is where artifacts for the base
- runtime environment should be staged.
+ runtime environment should be staged. Only works if
+ BST_VIRTUAL_DIRECTORY is not set.
Returns:
(str): The sandbox root directory
+
"""
- return self.__root
+ if self.__allow_real_directory:
+ return self.__root
+ else:
+ raise BstError("You can't use get_directory")
def set_environment(self, environment):
"""Sets the environment variables for the sandbox
[buildstream] 11/17: sandbox/_mount.py,
sandbox/_sandboxbwrap.py: Remove instances of get_directory
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 379ca4c496a1e354ea5f59f877f62f1a02d78a92
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:27:19 2018 +0100
sandbox/_mount.py, sandbox/_sandboxbwrap.py: Remove instances of get_directory
---
buildstream/sandbox/_mount.py | 3 ++-
buildstream/sandbox/_sandboxbwrap.py | 4 +++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/buildstream/sandbox/_mount.py b/buildstream/sandbox/_mount.py
index 84ab30a..225236d 100644
--- a/buildstream/sandbox/_mount.py
+++ b/buildstream/sandbox/_mount.py
@@ -33,7 +33,8 @@ from .._fuse import SafeHardlinks
class Mount():
def __init__(self, sandbox, mount_point, safe_hardlinks):
scratch_directory = sandbox._get_scratch_directory()
- root_directory = sandbox.get_directory()
+ # Getting external_directory here is acceptable as we're part of the sandbox code.
+ root_directory = sandbox.get_virtual_directory().external_directory
self.mount_point = mount_point
self.safe_hardlinks = safe_hardlinks
diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py
index d18cb9e..dc1b47d 100644
--- a/buildstream/sandbox/_sandboxbwrap.py
+++ b/buildstream/sandbox/_sandboxbwrap.py
@@ -56,7 +56,9 @@ class SandboxBwrap(Sandbox):
def run(self, command, flags, *, cwd=None, env=None):
stdout, stderr = self._get_output()
- root_directory = self.get_directory()
+
+ # Allowable access to underlying storage as we're part of the sandbox
+ root_directory = self.get_virtual_directory().external_directory
# Fallback to the sandbox default settings for
# the cwd and env.
[buildstream] 13/17: scriptelement.py: Convert to virtual
directories
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c52a5926f995b0acc633e091d53487213d056f20
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:22:21 2018 +0100
scriptelement.py: Convert to virtual directories
---
buildstream/scriptelement.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py
index 95e6928..6a1897a 100644
--- a/buildstream/scriptelement.py
+++ b/buildstream/scriptelement.py
@@ -245,9 +245,8 @@ class ScriptElement(Element):
with self.timed_activity("Staging {} at {}"
.format(element.name, item['destination']),
silent_nested=True):
- real_dstdir = os.path.join(sandbox.get_directory(),
- item['destination'].lstrip(os.sep))
- os.makedirs(os.path.dirname(real_dstdir), exist_ok=True)
+ virtual_dstdir = sandbox.get_virtual_directory()
+ virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True)
element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
for item in self.__layout:
@@ -265,8 +264,8 @@ class ScriptElement(Element):
for dep in element.dependencies(Scope.RUN):
dep.integrate(sandbox)
- os.makedirs(os.path.join(sandbox.get_directory(), self.__install_root.lstrip(os.sep)),
- exist_ok=True)
+ install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep)
+ sandbox.get_virtual_directory().descend(install_root_path_components, create=True)
def assemble(self, sandbox):
[buildstream] 14/17: Directory.py: Change the mtime interface into
mark_unmodified/list_modified_paths
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit fbde34e649d2acf19bcf77b35235cdacf5d8cb06
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 17:34:25 2018 +0100
Directory.py: Change the mtime interface into mark_unmodified/list_modified_paths
---
buildstream/plugins/elements/compose.py | 15 ++++++---------
buildstream/sandbox/Directory.py | 21 ++++++++++++++++-----
buildstream/sandbox/_filebaseddirectory.py | 29 ++++++++++++++++++++++++-----
3 files changed, 46 insertions(+), 19 deletions(-)
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index b6a1a29..aab0378 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -119,23 +119,20 @@ class ComposeElement(Element):
if require_split:
# Make a snapshot of all the files before integration-commands are run.
- snapshot = vbasedir.list_relative_paths_with_mtimes()
+ snapshot = vbasedir.list_relative_paths()
+ vbasedir.mark_unmodified()
for dep in self.dependencies(Scope.BUILD):
dep.integrate(sandbox)
if require_split:
# Calculate added, modified and removed files
- post_integration_snapshot = vbasedir.list_relative_paths_with_mtimes()
-
- basedir_contents = set(post_integration_snapshot.keys())
+ post_integration_snapshot = vbasedir.list_relative_paths()
+ modified_files = vbasedir.list_modified_paths()
+ basedir_contents = set(post_integration_snapshot)
for path in manifest:
if path in basedir_contents:
- if path in snapshot:
- preintegration_mtime = snapshot[path]
- if preintegration_mtime != post_integration_snapshot[path]:
- modified_files.add(path)
- else:
+ if path not in snapshot:
# If the path appears in the manifest but not the initial snapshot,
# it may be a file staged inside a directory symlink. In this case
# the path we got from the manifest won't show up in the snapshot
diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py
index 07f22ba..4107496 100644
--- a/buildstream/sandbox/Directory.py
+++ b/buildstream/sandbox/Directory.py
@@ -110,11 +110,22 @@ class Directory():
"""
raise NotImplementedError()
- def list_relative_paths_with_mtimes(self) -> Dict[str, float]:
- """Provide a list of relative paths with modification times for
- each. Used to detect changed changed files during a Compose
- operation.
+ def mark_unmodified(self) -> None:
+ """ Marks all files in this directory (recursively) as unmodified.
+ """
+ raise NotImplementedError()
+
+ def list_modified_paths(self) -> List[str]:
+ """Provide a list of relative paths which have been modified since the
+ last call to mark_unmodified.
+
+ Return value: List(str) - dictionary with all paths
+ """
+ raise NotImplementedError()
+
+ def list_relative_paths(self) -> List[str]:
+ """Provide a list of all relative paths in this directory.
- Return value: Dict(str->float) - dictionary with all paths and mtime in seconds.
+ Return value: List(str) - dictionary with all paths
"""
raise NotImplementedError()
diff --git a/buildstream/sandbox/_filebaseddirectory.py b/buildstream/sandbox/_filebaseddirectory.py
index 640e254..21a15fb 100644
--- a/buildstream/sandbox/_filebaseddirectory.py
+++ b/buildstream/sandbox/_filebaseddirectory.py
@@ -30,6 +30,7 @@ See also: :ref:`sandboxing`.
from typing import List
from collections import OrderedDict
+import calendar
import os
import time
from .._exceptions import BstError, ErrorDomain
@@ -210,11 +211,29 @@ class FileBasedDirectory(Directory):
self._populate_index()
return len(self.index) == 0
- def list_relative_paths_with_mtimes(self) -> Dict[str, float]:
- return {
- f: getmtime(os.path.join(self.external_directory, f))
- for f in list_relative_paths(self.external_directory)
- }
+ def mark_unmodified(self) -> None:
+ """ Marks all files in this directory (recursively) as unmodified.
+ """
+ _set_deterministic_mtime(self.external_directory)
+
+ def list_modified_paths(self) -> List[str]:
+ """Provide a list of relative paths which have been modified since the
+ last call to mark_unmodified.
+
+ Return value: List(str) - list of modified paths
+ """
+ magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11])
+
+ return [f for f in list_relative_paths(self.external_directory)
+ if getmtime(os.path.join(self.external_directory, f)) > magic_timestamp]
+
+ def list_relative_paths(self) -> List[str]:
+ """Provide a list of all relative paths.
+
+ Return value: List(str) - list of all paths
+ """
+
+ return list_relative_paths(self.external_directory)
def __str__(self) -> str:
# This returns the whole path (since we don't know where the directory started)
[buildstream] 01/17: Revert "Replace bwrap checks with calls to
check_bwrap_version"
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5034957ebbd4248be352b7b6be118f09f7d3cd4e
Author: Tristan Van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu May 10 20:52:15 2018 +0900
Revert "Replace bwrap checks with calls to check_bwrap_version"
This reverts commit f8952d6b8a775026d8a566969dd2570badf838fe.
For some reason, the changes introduced here cause issue #395 to
occur, without these changes we are not hitting the spurrious errors
described in #395.
---
buildstream/_platform/linux.py | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index 26dafb9..c4e87de 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -20,7 +20,6 @@
import subprocess
-from .. import _site
from .. import utils
from .._artifactcache.ostreecache import OSTreeCache
from .._message import Message, MessageType
@@ -35,8 +34,8 @@ class Linux(Platform):
super().__init__(context, project)
- self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
self._user_ns_available = self._check_user_ns_available(context)
+ self._die_with_parent_available = self._check_die_with_parent_available(context)
self._artifact_cache = OSTreeCache(context, enable_push=self._user_ns_available)
@property
@@ -82,3 +81,20 @@ class Linux(Platform):
detail="Some builds may not function due to lack of uid / gid 0, " +
"artifacts created will not be trusted for push purposes."))
return False
+
+ def _check_die_with_parent_available(self, context):
+
+ # bwrap supports --die-with-parent since 0.1.8.
+ # Let's check whether the host bwrap supports it.
+ bwrap = utils.get_host_tool('bwrap')
+
+ try:
+ subprocess.check_call([
+ bwrap,
+ '--ro-bind', '/', '/',
+ '--die-with-parent',
+ 'true'
+ ], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+ return True
+ except subprocess.CalledProcessError:
+ return False
[buildstream] 17/17: _stream.py: Alter _checkout_hardlinks to use
can_destroy
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a3dfc3fb9003b301fb7910c8986dc99e1b844d0d
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Wed May 9 14:01:23 2018 +0100
_stream.py: Alter _checkout_hardlinks to use can_destroy
---
buildstream/_stream.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index b7848c3..14a4a9e 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -927,12 +927,10 @@ class Stream():
raise StreamError("Failed to remove checkout directory: {}".format(e)) from e
if removed:
- # TODO: Direct rename is no longer possible with the new Virtual Directory interface.
- # See what options there are to restore it.
os.makedirs(directory, exist_ok=True)
- sandbox_vroot.export_files(directory, can_link=True)
+ sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
else:
- sandbox_vroot.export_files(directory, can_link=True)
+ sandbox_vroot.export_files(directory, can_link=True, can_destroy=False)
# Write the element build script to the given directory
def _write_element_script(self, directory, element):
[buildstream] 16/17: Implement can_destroy flag in
_filebaseddirectory.py
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2e8748a89bad059abf4fd4b218874ed44cd302a1
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Wed May 9 13:00:06 2018 +0100
Implement can_destroy flag in _filebaseddirectory.py
---
buildstream/sandbox/Directory.py | 5 ++++-
buildstream/sandbox/_filebaseddirectory.py | 13 ++++++++++++-
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py
index 0a60fce..f37fb98 100644
--- a/buildstream/sandbox/Directory.py
+++ b/buildstream/sandbox/Directory.py
@@ -80,7 +80,7 @@ class Directory():
raise NotImplementedError()
- def export_files(self, to_directory: str, can_link: bool = False) -> None:
+ def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None:
"""Copies everything from this into to_directory.
Arguments:
@@ -91,6 +91,9 @@ class Directory():
can_link (bool): Whether we can create hard links in to_directory
instead of copying. Setting this does not guarantee hard links will be used.
+ can_destroy (bool): Can we destroy the data already in this
+ directory when exporting? If set, this may allow data to be
+ moved rather than copied which will be quicker.
"""
raise NotImplementedError()
diff --git a/buildstream/sandbox/_filebaseddirectory.py b/buildstream/sandbox/_filebaseddirectory.py
index 21a15fb..fd27f8e 100644
--- a/buildstream/sandbox/_filebaseddirectory.py
+++ b/buildstream/sandbox/_filebaseddirectory.py
@@ -188,7 +188,7 @@ class FileBasedDirectory(Directory):
"""
_set_deterministic_user(self.external_directory)
- def export_files(self, to_directory: str, can_link: bool = False) -> None:
+ def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None:
"""Copies everything from this into to_directory.
Arguments:
@@ -200,6 +200,17 @@ class FileBasedDirectory(Directory):
instead of copying.
"""
+
+ if can_destroy:
+ # Try a simple rename of the sandbox root; if that
+ # doesnt cut it, then do the regular link files code path
+ try:
+ os.rename(self.external_directory, to_directory)
+ return
+ except OSError:
+ # Proceed using normal link/copy
+ pass
+
if can_link:
link_files(self.external_directory, to_directory)
else:
[buildstream] 08/17: element.py: Conversion to use virtual
directories
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 692d76a67c9476a9a677dc849e677be20873aeb0
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:19:22 2018 +0100
element.py: Conversion to use virtual directories
---
buildstream/element.py | 87 +++++++++++++++++++++++++-------------------------
1 file changed, 44 insertions(+), 43 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index c3c3513..7b827ea 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -81,7 +81,6 @@ from collections import Mapping, OrderedDict
from contextlib import contextmanager
from enum import Enum
import tempfile
-import time
import shutil
from . import _yaml
@@ -98,6 +97,9 @@ from . import _site
from ._platform import Platform
from .sandbox._config import SandboxConfig
+from .sandbox.Directory import Directory
+from .sandbox._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError
+
# _KeyStrength():
#
@@ -628,10 +630,10 @@ class Element(Plugin):
# Hard link it into the staging area
#
- basedir = sandbox.get_directory()
- stagedir = basedir \
+ vbasedir = sandbox.get_virtual_directory()
+ vstagedir = vbasedir \
if path is None \
- else os.path.join(basedir, path.lstrip(os.sep))
+ else vbasedir.descend(path.lstrip(os.sep).split(os.sep))
files = list(self.__compute_splits(include, exclude, orphans))
@@ -643,15 +645,8 @@ class Element(Plugin):
link_files = files
copy_files = []
- link_result = utils.link_files(artifact, stagedir, files=link_files,
- report_written=True)
- copy_result = utils.copy_files(artifact, stagedir, files=copy_files,
- report_written=True)
-
- cur_time = time.time()
-
- for f in copy_result.files_written:
- os.utime(os.path.join(stagedir, f), times=(cur_time, cur_time))
+ link_result = vstagedir.import_files(artifact, files=link_files, report_written=True, can_link=True)
+ copy_result = vstagedir.import_files(artifact, files=copy_files, report_written=True, update_utimes=True)
return link_result.combine(copy_result)
@@ -1312,8 +1307,8 @@ class Element(Plugin):
sandbox._set_mount_source(directory, workspace.get_absolute_path())
# Stage all sources that need to be copied
- sandbox_root = sandbox.get_directory()
- host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep))
+ sandbox_vroot = sandbox.get_virtual_directory()
+ host_directory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces)
# _stage_sources_at():
@@ -1324,28 +1319,33 @@ class Element(Plugin):
# directory (str): An absolute path to stage the sources at
# mount_workspaces (bool): mount workspaces if True, copy otherwise
#
- def _stage_sources_at(self, directory, mount_workspaces=True):
+ def _stage_sources_at(self, vdirectory, mount_workspaces=True):
with self.timed_activity("Staging sources", silent_nested=True):
- if os.path.isdir(directory) and os.listdir(directory):
- raise ElementError("Staging directory '{}' is not empty".format(directory))
-
- workspace = self._get_workspace()
- if workspace:
- # If mount_workspaces is set and we're doing incremental builds,
- # the workspace is already mounted into the sandbox.
- if not (mount_workspaces and self.__can_build_incrementally()):
- with self.timed_activity("Staging local files at {}".format(workspace.path)):
- workspace.stage(directory)
- else:
- # No workspace, stage directly
- for source in self.sources():
- source._stage(directory)
-
+ if not isinstance(vdirectory, Directory):
+ vdirectory = FileBasedDirectory(vdirectory)
+ if not vdirectory.is_empty():
+ raise ElementError("Staging directory '{}' is not empty".format(vdirectory))
+
+ with tempfile.TemporaryDirectory() as temp_staging_directory:
+
+ workspace = self._get_workspace()
+ if workspace:
+ # If mount_workspaces is set and we're doing incremental builds,
+ # the workspace is already mounted into the sandbox.
+ if not (mount_workspaces and self.__can_build_incrementally()):
+ with self.timed_activity("Staging local files at {}".format(workspace.path)):
+ workspace.stage(temp_staging_directory)
+ else:
+ # No workspace, stage directly
+ for source in self.sources():
+ source._stage(temp_staging_directory)
+
+ vdirectory.import_files(temp_staging_directory, None)
# Ensure deterministic mtime of sources at build time
- utils._set_deterministic_mtime(directory)
+ vdirectory.set_deterministic_mtime()
# Ensure deterministic owners of sources at build time
- utils._set_deterministic_user(directory)
+ vdirectory.set_deterministic_user()
# _schedule_assemble():
#
@@ -1422,7 +1422,7 @@ class Element(Plugin):
with _signals.terminator(cleanup_rootdir), \
self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # nopep8
- sandbox_root = sandbox.get_directory()
+ sandbox_vroot = sandbox.get_virtual_directory()
# By default, the dynamic public data is the same as the static public data.
# The plugin's assemble() method may modify this, though.
@@ -1452,23 +1452,24 @@ class Element(Plugin):
#
workspace = self._get_workspace()
if workspace and self.__staged_sources_directory:
- sandbox_root = sandbox.get_directory()
- sandbox_path = os.path.join(sandbox_root,
- self.__staged_sources_directory.lstrip(os.sep))
+ sandbox_vroot = sandbox.get_virtual_directory()
+ path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
+ sandbox_vpath = sandbox_vroot.descend(path_components)
try:
- utils.copy_files(workspace.path, sandbox_path)
+ sandbox_vpath.import_files(workspace.path)
except UtilError as e:
self.warn("Failed to preserve workspace state for failed build sysroot: {}"
.format(e))
raise
- collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep))
- if not os.path.exists(collectdir):
+ try:
+ collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
+ except VirtualDirectoryError:
raise ElementError(
- "Directory '{}' was not found inside the sandbox, "
+ "Subdirectory '{}' of '{}' does not exist following assembly, "
"unable to collect artifact contents"
- .format(collect))
+ .format(collect, sandbox_vroot))
# At this point, we expect an exception was raised leading to
# an error message, or we have good output to collect.
@@ -1484,7 +1485,7 @@ class Element(Plugin):
os.mkdir(metadir)
# Hard link files from collect dir to files directory
- utils.link_files(collectdir, filesdir)
+ collectvdir.export_files(filesdir, can_link=True)
# Copy build log
if self.__log_path:
[buildstream] 06/17: Sandbox.py: Add get_virtual_directory and
get_virtual_toplevel_directory
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7010dab38e6e645d64c2e6c334f54db71132d021
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:17:50 2018 +0100
Sandbox.py: Add get_virtual_directory and get_virtual_toplevel_directory
---
buildstream/sandbox/sandbox.py | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py
index c99bbaf..979367b 100644
--- a/buildstream/sandbox/sandbox.py
+++ b/buildstream/sandbox/sandbox.py
@@ -32,6 +32,7 @@ See also: :ref:`sandboxing`.
import os
from .._exceptions import ImplError, BstError
+from ._filebaseddirectory import FileBasedDirectory
class SandboxFlags():
@@ -123,6 +124,33 @@ class Sandbox():
else:
raise BstError("You can't use get_directory")
+ def get_virtual_directory(self):
+ """Fetches the sandbox root directory
+
+ The root directory is where artifacts for the base
+ runtime environment should be staged. Only works if
+ BST_VIRTUAL_DIRECTORY is not set.
+
+ Returns:
+ (str): The sandbox root directory
+
+ """
+ # For now, just create a new Directory every time we're asked
+ return FileBasedDirectory(self._root)
+
+ def get_virtual_toplevel_directory(self):
+ """Fetches the sandbox's toplevel directory
+
+ The toplevel directory contains 'root', 'scratch' and later
+ 'artifact' where output is copied to.
+
+ Returns:
+ (str): The sandbox toplevel directory
+
+ """
+ # For now, just create a new Directory every time we're asked
+ return FileBasedDirectory(self.__directory)
+
def set_environment(self, environment):
"""Sets the environment variables for the sandbox
[buildstream] 07/17: _stream.py: Convert to virtual directories.
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ad87b9230748491d7c6eaa5d09cad096b1b13652
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:18:36 2018 +0100
_stream.py: Convert to virtual directories.
---
buildstream/_stream.py | 21 +++++++++------------
1 file changed, 9 insertions(+), 12 deletions(-)
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index a5eacac..b7848c3 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -390,13 +390,13 @@ class Stream():
with target._prepare_sandbox(Scope.RUN, None, integrate=integrate) as sandbox:
# Copy or move the sandbox to the target directory
- sandbox_root = sandbox.get_directory()
+ sandbox_vroot = sandbox.get_virtual_directory()
with target.timed_activity("Checking out files in {}".format(directory)):
try:
if hardlinks:
- self._checkout_hardlinks(sandbox_root, directory)
+ self._checkout_hardlinks(sandbox_vroot, directory)
else:
- utils.copy_files(sandbox_root, directory)
+ sandbox_vroot.export_files(directory)
except OSError as e:
raise StreamError("Failed to checkout files: {}".format(e)) from e
except BstError as e:
@@ -920,22 +920,19 @@ class Stream():
# Helper function for checkout()
#
- def _checkout_hardlinks(self, sandbox_root, directory):
+ def _checkout_hardlinks(self, sandbox_vroot, directory):
try:
removed = utils.safe_remove(directory)
except OSError as e:
raise StreamError("Failed to remove checkout directory: {}".format(e)) from e
if removed:
- # Try a simple rename of the sandbox root; if that
- # doesnt cut it, then do the regular link files code path
- try:
- os.rename(sandbox_root, directory)
- except OSError:
- os.makedirs(directory, exist_ok=True)
- utils.link_files(sandbox_root, directory)
+ # TODO: Direct rename is no longer possible with the new Virtual Directory interface.
+ # See what options there are to restore it.
+ os.makedirs(directory, exist_ok=True)
+ sandbox_vroot.export_files(directory, can_link=True)
else:
- utils.link_files(sandbox_root, directory)
+ sandbox_vroot.export_files(directory, can_link=True)
# Write the element build script to the given directory
def _write_element_script(self, directory, element):
[buildstream] 04/17: Add the virtual directory class 'Directory'
and one implementation,
_filebaseddirectory.py. These are not used anywhere yet.
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9c436702a7e286150fa96096ae309162d6d1a3ab
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:08:12 2018 +0100
Add the virtual directory class 'Directory' and one implementation,
_filebaseddirectory.py. These are not used anywhere yet.
buildstream/sandbox/Directory.py: New file.
buildstream/sandbox/_filebaseddirectory.py: New file.
buildstream/_exceptions.py: New VIRTUAL_FS exception source.
---
buildstream/_exceptions.py | 1 +
buildstream/sandbox/Directory.py | 120 ++++++++++++++++
buildstream/sandbox/_filebaseddirectory.py | 223 +++++++++++++++++++++++++++++
3 files changed, 344 insertions(+)
diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py
index bcea65a..bcff8af 100644
--- a/buildstream/_exceptions.py
+++ b/buildstream/_exceptions.py
@@ -89,6 +89,7 @@ class ErrorDomain(Enum):
ELEMENT = 11
APP = 12
STREAM = 13
+ VIRTUAL_FS = 14
# BstError is an internal base exception class for BuildSream
diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py
new file mode 100644
index 0000000..07f22ba
--- /dev/null
+++ b/buildstream/sandbox/Directory.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Jim MacArthur <ji...@codethink.co.uk>
+
+"""
+Directory
+=========
+
+Virtual Directory class to isolate the rest of BuildStream from the backing store implementation.
+Sandboxes are allowed to read from and write to the underlying storage, but all others must use this
+Directory class to access files and directories in the sandbox.
+
+See also: :ref:`sandboxing`.
+"""
+
+from typing import List
+from ..utils import FileListResult
+
+
+class Directory():
+ def __init__(self, external_directory=None):
+ raise NotImplementedError()
+
+ def descend(self, subdirectory_spec: List[str]) -> 'Directory':
+ """
+ Descend one or more levels of directory hierarchy and return a new
+ Directory object for that directory.
+
+ Arguments:
+ subdirectory_spec (list of strings): A list of strings which are all directory
+ names.
+ create (boolean): If this is true, the directories will be created if
+ they don't already exist.
+ """
+ raise NotImplementedError()
+
+ # Import and export of files and links
+ def import_files(self, external_pathspec: any, files: List[str] = None,
+ report_written: bool = True, update_utimes: bool = False,
+ link_ok: bool = False) -> FileListResult:
+ """Imports some or all files from external_path into this directory.
+
+ Keyword arguments: external_pathspec: Either a string
+ containing a pathname, or a Directory object, to use as the
+ source.
+
+ files (list of strings): A list of all the files relative to
+ the external_pathspec to copy. If 'None' is supplied, all
+ files are copied.
+
+ report_written (bool): Return the full list of files
+ written. Defaults to true. If false, only a list of
+ overwritten files is returned.
+
+ update_utimes (bool): Update the access and modification time
+ of each file copied to the current time.
+
+ can_link (bool): Whether it's OK to create a hard link to the
+ original content, meaning the stored copy will change when the
+ original files change. Setting this doesn't guarantee hard
+ links will be made. can_link will never be used if
+ update_utimes is set.
+ """
+
+ raise NotImplementedError()
+
+ def export_files(self, to_directory: str, link_ok: bool = False) -> None:
+ """Copies everything from this into to_directory.
+
+ Arguments:
+
+ to_directory (string): a path outside this directory object
+ where the contents will be copied to.
+
+ can_link (bool): Whether we can create hard links in to_directory
+ instead of copying. Setting this does not guarantee hard links will be used.
+
+ """
+
+ raise NotImplementedError()
+
+ # Convenience functions
+ def is_empty(self) -> bool:
+ raise NotImplementedError()
+
+ def set_deterministic_mtime(self) -> None:
+ """ Sets a static modification time for all regular files in this directory.
+ The magic number for timestamps: 2011-11-11 11:11:11
+ """
+ raise NotImplementedError()
+
+ def set_deterministic_user(self) -> None:
+ """ Sets all files in this directory to the current user's euid/egid.
+ """
+ raise NotImplementedError()
+
+ def list_relative_paths_with_mtimes(self) -> Dict[str, float]:
+ """Provide a list of relative paths with modification times for
+ each. Used to detect changed changed files during a Compose
+ operation.
+
+ Return value: Dict(str->float) - dictionary with all paths and mtime in seconds.
+ """
+ raise NotImplementedError()
diff --git a/buildstream/sandbox/_filebaseddirectory.py b/buildstream/sandbox/_filebaseddirectory.py
new file mode 100644
index 0000000..640e254
--- /dev/null
+++ b/buildstream/sandbox/_filebaseddirectory.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Jim MacArthur <ji...@codethink.co.uk>
+
+"""
+FileBasedDirectory
+=========
+
+Implementation of the Directory class which backs onto a normal POSIX filing system.
+
+See also: :ref:`sandboxing`.
+"""
+
+from typing import List
+from collections import OrderedDict
+
+import os
+import time
+from .._exceptions import BstError, ErrorDomain
+from .Directory import Directory
+from ..utils import link_files, copy_files, FileListResult, list_relative_paths
+from ..utils import _set_deterministic_user, _set_deterministic_mtime
+
+
+class VirtualDirectoryError(BstError):
+ """Raised by Directory functions when system calls fail.
+ This will be handled internally by the BuildStream core,
+ if you need to handle this error, then it should be reraised,
+ or either of the :class:`.ElementError` or :class:`.SourceError`
+ exceptions should be raised from this error.
+ """
+ def __init__(self, message, reason=None):
+ super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason)
+
+
+# Like os.path.getmtime(), but doesnt explode on symlinks
+# Copy/pasted from compose.py
+def getmtime(path):
+ stat = os.lstat(path)
+ return stat.st_mtime
+
+# FileBasedDirectory intentionally doesn't call its superclass constuctor,
+# which is mean to be unimplemented.
+# pylint: disable=super-init-not-called
+
+
+class _FileObject():
+ """A description of a file in a virtual directory. The contents of
+ this class are never used, but there needs to be something present
+ for files so is_empty() works correctly.
+
+ """
+ def __init__(self, virtual_directory: Directory, filename: str):
+ self.directory = virtual_directory
+ self.filename = filename
+
+
+class FileBasedDirectory(Directory):
+ def __init__(self, external_directory=None):
+ self.external_directory = external_directory
+ self.index = OrderedDict()
+ self._directory_read = False
+
+ def _populate_index(self) -> None:
+ if self._directory_read:
+ return
+ for entry in os.listdir(self.external_directory):
+ if os.path.isdir(os.path.join(self.external_directory, entry)):
+ self.index[entry] = FileBasedDirectory(os.path.join(self.external_directory, entry))
+ else:
+ self.index[entry] = _FileObject(self, entry)
+ self._directory_read = True
+
+ def descend(self, subdirectory_spec: List[str], create: bool = False) -> Directory:
+ """ Descend one or more levels of directory hierarchy and return a new
+ Directory object for that directory.
+
+ Arguments:
+ * subdirectory_spec (list of strings): A list of strings which are all directory
+ names.
+ * create (boolean): If this is true, the directories will be created if
+ they don't already exist.
+ """
+
+ # It's very common to send a directory name instead of a list and this causes
+ # bizarre errors, so check for it here
+ if not isinstance(subdirectory_spec, list):
+ subdirectory_spec = [subdirectory_spec]
+ if not subdirectory_spec:
+ return self
+
+ # Because of the way split works, it's common to get a list which begins with
+ # an empty string. Detect these and remove them, then start again.
+ if subdirectory_spec[0] == "":
+ return self.descend(subdirectory_spec[1:], create)
+
+ self._populate_index()
+ if subdirectory_spec[0] in self.index:
+ entry = self.index[subdirectory_spec[0]]
+ if isinstance(entry, FileBasedDirectory):
+ new_path = os.path.join(self.external_directory, subdirectory_spec[0])
+ return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create)
+ else:
+ error = "Cannot descend into {}, which is a '{}' in the directory {}"
+ raise VirtualDirectoryError(error.format(subdirectory_spec[0],
+ type(entry).__name__,
+ self.external_directory))
+ else:
+ if create:
+ new_path = os.path.join(self.external_directory, subdirectory_spec[0])
+ os.makedirs(new_path, exist_ok=True)
+ return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create)
+ else:
+ error = "No entry called '{}' found in the directory rooted at {}"
+ raise VirtualDirectoryError(error.format(subdirectory_spec[0], self.external_directory))
+ return None
+
+ def import_files(self, external_pathspec: any, files: List[str] = None,
+ report_written: bool = True, update_utimes: bool = False,
+ can_link: bool = False) -> FileListResult:
+ """Imports some or all files from external_path into this directory.
+
+ Keyword arguments: external_pathspec: Either a string
+ containing a pathname, or a Directory object, to use as the
+ source.
+
+ files (list of strings): A list of all the files relative to
+ the external_pathspec to copy. If 'None' is supplied, all
+ files are copied.
+
+ report_written (bool): Return the full list of files
+ written. Defaults to true. If false, only a list of
+ overwritten files is returned.
+
+ update_utimes (bool): Update the access and modification time
+ of each file copied to the current time.
+
+ can_link (bool): Whether it's OK to create a hard link to the
+ original content, meaning the stored copy will change when the
+ original files change. Setting this doesn't guarantee hard
+ links will be made. can_link will never be used if
+ update_utimes is set.
+ """
+
+ if isinstance(external_pathspec, Directory):
+ source_directory = external_pathspec.external_directory
+ else:
+ source_directory = external_pathspec
+
+ if can_link and not update_utimes:
+ import_result = link_files(source_directory, self.external_directory, files=files,
+ ignore_missing=False, report_written=report_written)
+ else:
+ import_result = copy_files(source_directory, self.external_directory, files=files,
+ ignore_missing=False, report_written=report_written)
+ if update_utimes:
+ cur_time = time.time()
+
+ for f in import_result.files_written:
+ os.utime(os.path.join(self.external_directory, f), times=(cur_time, cur_time))
+ return import_result
+
+ def set_deterministic_mtime(self) -> None:
+ """ Sets a static modification time for all regular files in this directory.
+ The magic number for timestamps: 2011-11-11 11:11:11
+ """
+ _set_deterministic_mtime(self.external_directory)
+
+ def set_deterministic_user(self) -> None:
+ """ Sets all files in this directory to the current user's euid/egid.
+ """
+ _set_deterministic_user(self.external_directory)
+
+ def export_files(self, to_directory: str, can_link: bool = False) -> None:
+ """Copies everything from this into to_directory.
+
+ Arguments:
+
+ to_directory (string): a path outside this directory object
+ where the contents will be copied to.
+
+ can_link (bool): Whether we can create hard links in to_directory
+ instead of copying.
+
+ """
+ if can_link:
+ link_files(self.external_directory, to_directory)
+ else:
+ copy_files(self.external_directory, to_directory)
+
+ def is_empty(self) -> bool:
+ """ Return true if this directory has no files, subdirectories or links in it.
+ """
+ self._populate_index()
+ return len(self.index) == 0
+
+ def list_relative_paths_with_mtimes(self) -> Dict[str, float]:
+ return {
+ f: getmtime(os.path.join(self.external_directory, f))
+ for f in list_relative_paths(self.external_directory)
+ }
+
+ def __str__(self) -> str:
+ # This returns the whole path (since we don't know where the directory started)
+ # which exposes the sandbox directory; we will have to assume for the time being
+ # that people will not abuse __str__.
+ return self.external_directory
[buildstream] 10/17: plugins/elements/import.py: Convert to virtual
directories
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ca097141a203f130c54c9b460284a197d09a9f7c
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:20:55 2018 +0100
plugins/elements/import.py: Convert to virtual directories
---
buildstream/plugins/elements/import.py | 20 +++++++-------------
1 file changed, 7 insertions(+), 13 deletions(-)
diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py
index 747455d..8b701c7 100644
--- a/buildstream/plugins/elements/import.py
+++ b/buildstream/plugins/elements/import.py
@@ -31,7 +31,6 @@ The empty configuration is as such:
"""
import os
-import shutil
from buildstream import Element, BuildElement, ElementError
@@ -68,27 +67,22 @@ class ImportElement(BuildElement):
# Do not mount workspaces as the files are copied from outside the sandbox
self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False)
- rootdir = sandbox.get_directory()
- inputdir = os.path.join(rootdir, 'input')
- outputdir = os.path.join(rootdir, 'output')
+ rootdir = sandbox.get_virtual_directory()
+ inputdir = rootdir.descend(['input'])
+ outputdir = rootdir.descend(['output'], create=True)
# The directory to grab
- inputdir = os.path.join(inputdir, self.source.lstrip(os.sep))
- inputdir = inputdir.rstrip(os.sep)
+ inputdir = inputdir.descend(self.source.strip(os.sep).split(os.sep))
# The output target directory
- outputdir = os.path.join(outputdir, self.target.lstrip(os.sep))
- outputdir = outputdir.rstrip(os.sep)
-
- # Ensure target directory parent
- os.makedirs(os.path.dirname(outputdir), exist_ok=True)
+ outputdir = outputdir.descend(self.target.strip(os.sep).split(os.sep), create=True)
- if not os.path.exists(inputdir):
+ if inputdir.is_empty():
raise ElementError("{}: No files were found inside directory '{}'"
.format(self, self.source))
# Move it over
- shutil.move(inputdir, outputdir)
+ outputdir.import_files(inputdir)
# And we're done
return '/output'
[buildstream] 12/17: plugins/elements/stack.py: Convert to virtual
directories
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b842699928c3be50ff9888df7899c65707b1258a
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:27:45 2018 +0100
plugins/elements/stack.py: Convert to virtual directories
---
buildstream/plugins/elements/stack.py | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/buildstream/plugins/elements/stack.py b/buildstream/plugins/elements/stack.py
index 45c49c5..5b237d2 100644
--- a/buildstream/plugins/elements/stack.py
+++ b/buildstream/plugins/elements/stack.py
@@ -24,7 +24,6 @@ Stack elements are simply a symbolic element used for representing
a logical group of elements.
"""
-import os
from buildstream import Element
@@ -52,7 +51,7 @@ class StackElement(Element):
# Just create a dummy empty artifact, its existence is a statement
# that all this stack's dependencies are built.
- rootdir = sandbox.get_directory()
+ vrootdir = sandbox.get_virtual_directory()
# XXX FIXME: This is currently needed because the artifact
# cache wont let us commit an empty artifact.
@@ -61,10 +60,7 @@ class StackElement(Element):
# the actual artifact data in a subdirectory, then we
# will be able to store some additional state in the
# artifact cache, and we can also remove this hack.
- outputdir = os.path.join(rootdir, 'output', 'bst')
-
- # Ensure target directory parent
- os.makedirs(os.path.dirname(outputdir), exist_ok=True)
+ vrootdir.descend(['output', 'bst'], create=True)
# And we're done
return '/output'
[buildstream] 09/17: plugins/elements/compose.py: Convert to
virtual directories
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f9a3b53c1502108151a96e11fd268efe42a7c6de
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:20:41 2018 +0100
plugins/elements/compose.py: Convert to virtual directories
---
buildstream/plugins/elements/compose.py | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index 0e666c6..b6a1a29 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -34,7 +34,6 @@ The default configuration and possible options are as such:
"""
import os
-from buildstream import utils
from buildstream import Element, Scope
@@ -56,6 +55,9 @@ class ComposeElement(Element):
# added, to reduce the potential for confusion
BST_FORBID_SOURCES = True
+ # This plugin has been modified to avoid the use of Sandbox.get_directory
+ BST_VIRTUAL_DIRECTORY = True
+
def configure(self, node):
self.node_validate(node, [
'integrate', 'include', 'exclude', 'include-orphans'
@@ -104,7 +106,8 @@ class ComposeElement(Element):
orphans=self.include_orphans)
manifest.update(files)
- basedir = sandbox.get_directory()
+ # Make a snapshot of all the files.
+ vbasedir = sandbox.get_virtual_directory()
modified_files = set()
removed_files = set()
added_files = set()
@@ -116,23 +119,21 @@ class ComposeElement(Element):
if require_split:
# Make a snapshot of all the files before integration-commands are run.
- snapshot = {
- f: getmtime(os.path.join(basedir, f))
- for f in utils.list_relative_paths(basedir)
- }
+ snapshot = vbasedir.list_relative_paths_with_mtimes()
for dep in self.dependencies(Scope.BUILD):
dep.integrate(sandbox)
if require_split:
-
# Calculate added, modified and removed files
- basedir_contents = set(utils.list_relative_paths(basedir))
+ post_integration_snapshot = vbasedir.list_relative_paths_with_mtimes()
+
+ basedir_contents = set(post_integration_snapshot.keys())
for path in manifest:
if path in basedir_contents:
if path in snapshot:
preintegration_mtime = snapshot[path]
- if preintegration_mtime != getmtime(os.path.join(basedir, path)):
+ if preintegration_mtime != post_integration_snapshot[path]:
modified_files.add(path)
else:
# If the path appears in the manifest but not the initial snapshot,
@@ -166,8 +167,7 @@ class ComposeElement(Element):
# instead of into a subdir. The element assemble() method should
# support this in some way.
#
- installdir = os.path.join(basedir, 'buildstream', 'install')
- os.makedirs(installdir, exist_ok=True)
+ installdir = vbasedir.descend(['buildstream', 'install'], create=True)
# We already saved the manifest for created files in the integration phase,
# now collect the rest of the manifest.
@@ -191,7 +191,7 @@ class ComposeElement(Element):
with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
self.info("Composing {} files".format(len(manifest)))
- utils.link_files(basedir, installdir, files=manifest)
+ installdir.import_files(vbasedir, files=manifest, can_link=True)
# And we're done
return os.path.join(os.sep, 'buildstream', 'install')
[buildstream] 02/17: Revert "_site.py: Add check_bwrap_version()
function"
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c5d1a5a2b960ac8edf8a55fc009c3ed58204c11f
Author: Tristan Van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu May 10 20:52:24 2018 +0900
Revert "_site.py: Add check_bwrap_version() function"
This reverts commit 03823d124f61c18346dd2d8282055ab25d6f9aa6.
For some reason, the changes introduced here cause issue #395 to
occur, without these changes we are not hitting the spurrious errors
described in #395.
---
buildstream/_site.py | 49 -------------------------------------------------
1 file changed, 49 deletions(-)
diff --git a/buildstream/_site.py b/buildstream/_site.py
index f4780ef..6228395 100644
--- a/buildstream/_site.py
+++ b/buildstream/_site.py
@@ -19,8 +19,6 @@
# Tristan Van Berkom <tr...@codethink.co.uk>
import os
-import shutil
-import subprocess
#
# Private module declaring some info about where the buildstream
@@ -47,50 +45,3 @@ build_all_template = os.path.join(root, 'data', 'build-all.sh.in')
# Module building script template
build_module_template = os.path.join(root, 'data', 'build-module.sh.in')
-
-# Cached bwrap version
-_bwrap_major = None
-_bwrap_minor = None
-_bwrap_patch = None
-
-
-# check_bwrap_version()
-#
-# Checks the version of installed bwrap against the requested version
-#
-# Args:
-# major (int): The required major version
-# minor (int): The required minor version
-# patch (int): The required patch level
-#
-# Returns:
-# (bool): Whether installed bwrap meets the requirements
-#
-def check_bwrap_version(major, minor, patch):
- # pylint: disable=global-statement
-
- global _bwrap_major
- global _bwrap_minor
- global _bwrap_patch
-
- # Parse bwrap version and save into cache, if not already cached
- if _bwrap_major is None:
- bwrap_path = shutil.which('bwrap')
- if not bwrap_path:
- return False
- cmd = [bwrap_path, "--version"]
- version = str(subprocess.check_output(cmd).split()[1], "utf-8")
- _bwrap_major, _bwrap_minor, _bwrap_patch = map(int, version.split("."))
-
- # Check whether the installed version meets the requirements
- if _bwrap_major > major:
- return True
- elif _bwrap_major < major:
- return False
- else:
- if _bwrap_minor > minor:
- return True
- elif _bwrap_minor < minor:
- return False
- else:
- return _bwrap_patch >= patch