You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:25:00 UTC
[buildstream] branch sam/compose-log-splits created (now f50042e)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch sam/compose-log-splits
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at f50042e WIP
This branch includes the following new commits:
new f37a8f7 plugin.py: Add log() method
new f508685 Log details of artifact splitting when building 'compose' elements
new f50042e WIP
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/03: plugin.py: Add log() method
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch sam/compose-log-splits
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f37a8f7033fa28539082b6d30592baaf51881bf2
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Wed Nov 8 12:58:21 2017 +0000
plugin.py: Add log() method
This is a helper to log messages into the plugin's log file.
---
buildstream/plugin.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/buildstream/plugin.py b/buildstream/plugin.py
index bfa37ef..2521807 100644
--- a/buildstream/plugin.py
+++ b/buildstream/plugin.py
@@ -416,6 +416,18 @@ class Plugin():
"""
self.__message(MessageType.LOG, brief, detail=detail)
+ def log(self, brief, detail=None):
+ """Log a message into the plugin's log file
+
+ The message will not be shown in the master log at all (so it will not
+ be displayed to the user on the console).
+
+ Args:
+ brief (str): The brief message
+ detail (str): An optional detailed message, can be multiline output
+ """
+ self.__message(MessageType.LOG, brief, detail=detail)
+
@contextmanager
def timed_activity(self, activity_name, *, detail=None, silent_nested=False):
"""Context manager for performing timed activities in plugins
[buildstream] 03/03: WIP
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch sam/compose-log-splits
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f50042eff26fe7e2d21e7aa04db267f6d53e07f5
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Fri Feb 23 17:37:58 2018 +0000
WIP
---
buildstream/element.py | 8 +-----
buildstream/plugins/elements/compose.py | 48 ++++++++++++++++-----------------
2 files changed, 24 insertions(+), 32 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index f92b241..ac76010 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -535,13 +535,6 @@ class Element(Plugin):
raise ElementError("Non-whitelisted overlaps detected and fail-on-overlaps is set",
detail=error_detail, reason="overlap-error")
- if overwrites:
- detail = "Staged files overwrite existing files in staging area:\n"
- for key, value in overwrites.items():
- detail += "\nFrom {}:\n".format(key)
- detail += " " + " ".join(["/" + f + "\n" for f in value])
- self.warn("Overlapping files", detail=detail)
-
if ignored:
detail = "Not staging files which would replace non-empty directories:\n"
for key, value in ignored.items():
@@ -1858,6 +1851,7 @@ class Element(Plugin):
include_file = True
included_by_domains.append(domain)
if domain in exclude:
+ print("Exclude {} due to {}".format(filename, domain))
exclude_file = True
if orphans and not claimed_file:
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index 7b6f99c..9b42bd5 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -98,14 +98,16 @@ class ComposeElement(Element):
with self.timed_activity("Staging dependencies", silent_nested=True):
self.stage_dependency_artifacts(sandbox, Scope.BUILD)
- manifest = set()
+ file_list = set()
+ artifact_map = dict()
if require_split:
with self.timed_activity("Computing split", silent_nested=True):
for dep in self.dependencies(Scope.BUILD):
- files = dep.compute_manifest(include=self.include,
+ manifest = dep.compute_manifest(include=self.include,
exclude=self.exclude,
orphans=self.include_orphans)
- manifest.update(files)
+ file_list.update(manifest.keys())
+ artifact_map.update(manifest)
# Make a snapshot of all the files.
basedir = sandbox.get_directory()
@@ -128,8 +130,10 @@ class ComposeElement(Element):
if require_split:
seen = set()
+ print("\n\n\nsnapshot: {}\n\n\n".format(snapshot))
# Calculate added modified files
for path in utils.list_relative_paths(basedir):
+ print("Got: {}".format(path))
seen.add(path)
if snapshot.get(path) is None:
added_files.append(path)
@@ -138,7 +142,7 @@ class ComposeElement(Element):
# Calculate removed files
removed_files = [
- path for path in manifest
+ path for path in file_list
if path not in seen
]
self.info("Integration modified {}, added {} and removed {} files"
@@ -152,8 +156,10 @@ class ComposeElement(Element):
# Do we want to force include files which were modified by
# the integration commands, even if they were not added ?
#
- manifest.update(added_files)
- manifest.difference_update(removed_files)
+ file_list.update(added_files)
+ file_list.difference_update(removed_files)
+
+ print("Explicitly removeD: {}".format(removed_files))
# XXX We should be moving things outside of the build sandbox
# instead of into a subdir. The element assemble() method should
@@ -182,23 +188,13 @@ class ComposeElement(Element):
detail = "\n".join(lines)
+ total_files = len([f for f in file_list if f != '.'])
+
with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
- manifest = self.stage_dependency_artifacts(sandbox, Scope.BUILD,
- path=stagedir,
- include=self.include,
- exclude=self.exclude,
- orphans=self.include_orphans)
-
- if self.integration:
- self.status("Moving {} integration files".format(len(integration_files)))
- utils.move_files(basedir, installdir, integration_files)
-
- for filename in integration_files:
- manifest[filename] = manifest.get(filename, {})
- manifest[filename]['integration'] = True
-
- total_files = len(manifest)
- detail = self._readable_manifest(manifest)
+ self.info("Composing {} files".format(total_files))
+ utils.link_files(basedir, installdir, files=file_list)
+
+ detail = self._readable_manifest(file_list, artifact_map)
self.log("Composed {} files".format(total_files), detail=detail)
# And we're done
@@ -206,15 +202,17 @@ class ComposeElement(Element):
# Show a list of files that made it into the artifact, grouped by the
# artifact and split-rules domains that resulted in each one being there.
- def _readable_manifest(self, manifest):
+ def _readable_manifest(self, file_list, artifact_map):
domains = collections.defaultdict(list)
# Convert the filename->domain mapping into a domain->filename mapping.
- for filename, entry in manifest.items():
+ for filename in file_list:
+ print("filename: {}, map: {}".format(filename, artifact_map.get(filename)))
if filename == '.':
continue
- if 'artifact' in entry:
+ if filename in artifact_map:
+ entry = artifact_map[filename]
domains_for_file = entry.get('domains') or ["(no domain)"]
for domain in domains_for_file:
full_domain_name = entry['artifact'].name + " " + domain
[buildstream] 02/03: Log details of artifact splitting when
building 'compose' elements
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch sam/compose-log-splits
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f508685a3f5f77ee551cb27554186c679bd7911f
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Tue Nov 7 13:12:53 2017 +0000
Log details of artifact splitting when building 'compose' elements
Artifact split rules are complicated to reason about. There needs to be
some way of seeing why a 'compose' artifact included a given file.
The log file now contains a long message like this, listing every
artifact, domain and file:
[--:--:--] INFO [initramfs/initramfs.bst]: Composed 2369 files
Integration
- var/cache/ldconfig/aux-cache
gnu-toolchain/binutils.bst runtime
- usr/x86_64-unknown-linux-gnu
- usr/x86_64-unknown-linux-gnu/bin/ar
- usr/x86_64-unknown-linux-gnu/bin/as
- usr/x86_64-unknown-linux-gnu/bin/ld
gnu-toolchain/make.bst (not in any domain)
- usr
- usr/lib
- usr/share
...
Note that a file can be in multiple domains.
Size of the log files is a concern and it may be that in future we
'compress' some of these entries, e.g. if all files in a given
directory come from one artifact then we just need to list the
directory, not every filename.
---
buildstream/element.py | 50 +++++++++++++++++++++++++++----
buildstream/plugins/elements/compose.py | 52 +++++++++++++++++++++++++++++++--
2 files changed, 94 insertions(+), 8 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index f0df03b..f92b241 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -102,6 +102,20 @@ class ElementError(BstError):
super().__init__(message, detail=detail, domain=ErrorDomain.ELEMENT, reason=reason)
+class StagingResult():
+ """Result of a staging operation."""
+
+ def __init__(self, file_list_result=None, manifest=None):
+ if file_list_result:
+ # This class extends utils.FileListResult; but Python has no magic
+ # way of casting an object to a subclass, so we make a new object
+ # and copy all the attributes across from the old one.
+ self.__dict__.update(file_list_result.__dict__)
+
+ self.manifest = manifest or {}
+ """Map from each file to the artifact and split rule which staged it."""
+
+
class Element(Plugin):
"""Element()
@@ -430,9 +444,10 @@ class Element(Plugin):
if path is None \
else os.path.join(basedir, path.lstrip(os.sep))
- files = self.__compute_splits(include, exclude, orphans)
- result = utils.link_files(artifact, stagedir, files=files,
+ manifest = self.__compute_splits(include, exclude, orphans)
+ result = utils.link_files(artifact, stagedir, files=manifest.keys(),
report_written=True)
+ result = StagingResult(result, manifest)
return result
@@ -453,6 +468,10 @@ class Element(Plugin):
exclude (list): An optional list of domains to exclude files from
orphans (bool): Whether to include files not spoken for by split domains
+ Returns:
+ (dict): A mapping from each file to the artifact and split-rules domain
+ that produced it.
+
Raises:
(:class:`.ElementError`): If any of the dependencies in `scope` have not
yet produced artifacts, or if forbidden overlaps
@@ -461,7 +480,7 @@ class Element(Plugin):
ignored = {}
overlaps = OrderedDict()
files_written = {}
-
+ manifest = {}
for dep in self.dependencies(scope):
result = dep.stage_artifact(sandbox,
path=path,
@@ -483,6 +502,10 @@ class Element(Plugin):
if result.ignored:
ignored[dep.name] = result.ignored
+ for f in result.ignored:
+ del result.manifest[f]
+
+ manifest.update(result.manifest)
if overlaps:
overlap_error = overlap_warning = False
@@ -512,6 +535,13 @@ class Element(Plugin):
raise ElementError("Non-whitelisted overlaps detected and fail-on-overlaps is set",
detail=error_detail, reason="overlap-error")
+ if overwrites:
+ detail = "Staged files overwrite existing files in staging area:\n"
+ for key, value in overwrites.items():
+ detail += "\nFrom {}:\n".format(key)
+ detail += " " + " ".join(["/" + f + "\n" for f in value])
+ self.warn("Overlapping files", detail=detail)
+
if ignored:
detail = "Not staging files which would replace non-empty directories:\n"
for key, value in ignored.items():
@@ -519,6 +549,8 @@ class Element(Plugin):
detail += " " + " ".join(["/" + f + "\n" for f in value])
self.warn("Ignored files", detail=detail)
+ return manifest
+
def integrate(self, sandbox):
"""Integrate currently staged filesystem against this artifact.
@@ -1782,12 +1814,13 @@ class Element(Plugin):
def __compute_splits(self, include=None, exclude=None, orphans=True):
basedir = os.path.join(self.__artifacts.extract(self), 'files')
+ manifest = {}
# No splitting requested, just report complete artifact
if orphans and not (include or exclude):
for filename in utils.list_relative_paths(basedir):
- yield filename
- return
+ manifest[filename] = {'artifact': self}
+ return manifest
if not self.__splits:
self.__init_splits()
@@ -1816,12 +1849,14 @@ class Element(Plugin):
include_file = False
exclude_file = False
claimed_file = False
+ included_by_domains = []
for domain in element_domains:
if self.__splits[domain].match(filename):
claimed_file = True
if domain in include:
include_file = True
+ included_by_domains.append(domain)
if domain in exclude:
exclude_file = True
@@ -1829,7 +1864,10 @@ class Element(Plugin):
include_file = True
if include_file and not exclude_file:
- yield filename.lstrip(os.sep)
+ manifest_entry = {'artifact': self}
+ manifest_entry['domains'] = included_by_domains
+ manifest[filename.lstrip(os.sep)] = manifest_entry
+ return manifest
def _load_public_data(self):
self._assert_cached()
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index 29e289a..7b6f99c 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -33,6 +33,7 @@ The default configuration and possible options are as such:
:language: yaml
"""
+import collections
import os
from buildstream import utils
from buildstream import Element, ElementError, Scope
@@ -112,6 +113,7 @@ class ComposeElement(Element):
f: getmtime(os.path.join(basedir, f))
for f in utils.list_relative_paths(basedir)
}
+
modified_files = []
removed_files = []
added_files = []
@@ -181,12 +183,58 @@ class ComposeElement(Element):
detail = "\n".join(lines)
with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
- self.info("Composing {} files".format(len(manifest)))
- utils.link_files(basedir, installdir, files=manifest)
+ manifest = self.stage_dependency_artifacts(sandbox, Scope.BUILD,
+ path=stagedir,
+ include=self.include,
+ exclude=self.exclude,
+ orphans=self.include_orphans)
+
+ if self.integration:
+ self.status("Moving {} integration files".format(len(integration_files)))
+ utils.move_files(basedir, installdir, integration_files)
+
+ for filename in integration_files:
+ manifest[filename] = manifest.get(filename, {})
+ manifest[filename]['integration'] = True
+
+ total_files = len(manifest)
+ detail = self._readable_manifest(manifest)
+ self.log("Composed {} files".format(total_files), detail=detail)
# And we're done
return os.path.join(os.sep, 'buildstream', 'install')
+ # Show a list of files that made it into the artifact, grouped by the
+ # artifact and split-rules domains that resulted in each one being there.
+ def _readable_manifest(self, manifest):
+ domains = collections.defaultdict(list)
+
+ # Convert the filename->domain mapping into a domain->filename mapping.
+ for filename, entry in manifest.items():
+ if filename == '.':
+ continue
+
+ if 'artifact' in entry:
+ domains_for_file = entry.get('domains') or ["(no domain)"]
+ for domain in domains_for_file:
+ full_domain_name = entry['artifact'].name + " " + domain
+ if entry.get('integration', False) is True:
+ full_domain_name += " (modified during integration)"
+
+ domains[full_domain_name].append(filename)
+ else:
+ domains["Integration"].append(filename)
+
+ # Display the mapping neatly for the user.
+ lines = []
+ for domain in sorted(domains):
+ lines.extend(["", domain])
+
+ contents = sorted(domains[domain])
+ lines.extend(" - " + filename for filename in contents)
+
+ return "\n".join(lines)
+
# Like os.path.getmtime(), but doesnt explode on symlinks
#