You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:47:10 UTC
[buildstream] 08/17: element.py: Conversion to use virtual
directories
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jmac/virtual_directories_test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 692d76a67c9476a9a677dc849e677be20873aeb0
Author: Jim MacArthur <ji...@codethink.co.uk>
AuthorDate: Tue May 8 16:19:22 2018 +0100
element.py: Conversion to use virtual directories
---
buildstream/element.py | 87 +++++++++++++++++++++++++-------------------------
1 file changed, 44 insertions(+), 43 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index c3c3513..7b827ea 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -81,7 +81,6 @@ from collections import Mapping, OrderedDict
from contextlib import contextmanager
from enum import Enum
import tempfile
-import time
import shutil
from . import _yaml
@@ -98,6 +97,9 @@ from . import _site
from ._platform import Platform
from .sandbox._config import SandboxConfig
+from .sandbox.Directory import Directory
+from .sandbox._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError
+
# _KeyStrength():
#
@@ -628,10 +630,10 @@ class Element(Plugin):
# Hard link it into the staging area
#
- basedir = sandbox.get_directory()
- stagedir = basedir \
+ vbasedir = sandbox.get_virtual_directory()
+ vstagedir = vbasedir \
if path is None \
- else os.path.join(basedir, path.lstrip(os.sep))
+ else vbasedir.descend(path.lstrip(os.sep).split(os.sep))
files = list(self.__compute_splits(include, exclude, orphans))
@@ -643,15 +645,8 @@ class Element(Plugin):
link_files = files
copy_files = []
- link_result = utils.link_files(artifact, stagedir, files=link_files,
- report_written=True)
- copy_result = utils.copy_files(artifact, stagedir, files=copy_files,
- report_written=True)
-
- cur_time = time.time()
-
- for f in copy_result.files_written:
- os.utime(os.path.join(stagedir, f), times=(cur_time, cur_time))
+ link_result = vstagedir.import_files(artifact, files=link_files, report_written=True, can_link=True)
+ copy_result = vstagedir.import_files(artifact, files=copy_files, report_written=True, update_utimes=True)
return link_result.combine(copy_result)
@@ -1312,8 +1307,8 @@ class Element(Plugin):
sandbox._set_mount_source(directory, workspace.get_absolute_path())
# Stage all sources that need to be copied
- sandbox_root = sandbox.get_directory()
- host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep))
+ sandbox_vroot = sandbox.get_virtual_directory()
+ host_directory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces)
# _stage_sources_at():
@@ -1324,28 +1319,33 @@ class Element(Plugin):
# directory (str): An absolute path to stage the sources at
# mount_workspaces (bool): mount workspaces if True, copy otherwise
#
- def _stage_sources_at(self, directory, mount_workspaces=True):
+ def _stage_sources_at(self, vdirectory, mount_workspaces=True):
with self.timed_activity("Staging sources", silent_nested=True):
- if os.path.isdir(directory) and os.listdir(directory):
- raise ElementError("Staging directory '{}' is not empty".format(directory))
-
- workspace = self._get_workspace()
- if workspace:
- # If mount_workspaces is set and we're doing incremental builds,
- # the workspace is already mounted into the sandbox.
- if not (mount_workspaces and self.__can_build_incrementally()):
- with self.timed_activity("Staging local files at {}".format(workspace.path)):
- workspace.stage(directory)
- else:
- # No workspace, stage directly
- for source in self.sources():
- source._stage(directory)
-
+ if not isinstance(vdirectory, Directory):
+ vdirectory = FileBasedDirectory(vdirectory)
+ if not vdirectory.is_empty():
+ raise ElementError("Staging directory '{}' is not empty".format(vdirectory))
+
+ with tempfile.TemporaryDirectory() as temp_staging_directory:
+
+ workspace = self._get_workspace()
+ if workspace:
+ # If mount_workspaces is set and we're doing incremental builds,
+ # the workspace is already mounted into the sandbox.
+ if not (mount_workspaces and self.__can_build_incrementally()):
+ with self.timed_activity("Staging local files at {}".format(workspace.path)):
+ workspace.stage(temp_staging_directory)
+ else:
+ # No workspace, stage directly
+ for source in self.sources():
+ source._stage(temp_staging_directory)
+
+ vdirectory.import_files(temp_staging_directory, None)
# Ensure deterministic mtime of sources at build time
- utils._set_deterministic_mtime(directory)
+ vdirectory.set_deterministic_mtime()
# Ensure deterministic owners of sources at build time
- utils._set_deterministic_user(directory)
+ vdirectory.set_deterministic_user()
# _schedule_assemble():
#
@@ -1422,7 +1422,7 @@ class Element(Plugin):
with _signals.terminator(cleanup_rootdir), \
self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # nopep8
- sandbox_root = sandbox.get_directory()
+ sandbox_vroot = sandbox.get_virtual_directory()
# By default, the dynamic public data is the same as the static public data.
# The plugin's assemble() method may modify this, though.
@@ -1452,23 +1452,24 @@ class Element(Plugin):
#
workspace = self._get_workspace()
if workspace and self.__staged_sources_directory:
- sandbox_root = sandbox.get_directory()
- sandbox_path = os.path.join(sandbox_root,
- self.__staged_sources_directory.lstrip(os.sep))
+ sandbox_vroot = sandbox.get_virtual_directory()
+ path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
+ sandbox_vpath = sandbox_vroot.descend(path_components)
try:
- utils.copy_files(workspace.path, sandbox_path)
+ sandbox_vpath.import_files(workspace.path)
except UtilError as e:
self.warn("Failed to preserve workspace state for failed build sysroot: {}"
.format(e))
raise
- collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep))
- if not os.path.exists(collectdir):
+ try:
+ collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
+ except VirtualDirectoryError:
raise ElementError(
- "Directory '{}' was not found inside the sandbox, "
+ "Subdirectory '{}' of '{}' does not exist following assembly, "
"unable to collect artifact contents"
- .format(collect))
+ .format(collect, sandbox_vroot))
# At this point, we expect an exception was raised leading to
# an error message, or we have good output to collect.
@@ -1484,7 +1485,7 @@ class Element(Plugin):
os.mkdir(metadir)
# Hard link files from collect dir to files directory
- utils.link_files(collectdir, filesdir)
+ collectvdir.export_files(filesdir, can_link=True)
# Copy build log
if self.__log_path: