[buildstream] 01/06: Introducing caching of build trees

[tv...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch issue-21_Caching_build_trees
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 3bb30be2866718f909be0f914731574e7ff4c5b3
Author: Phillip Smyth <ph...@codethink.co.uk>
AuthorDate: Tue Mar 20 15:29:23 2018 +0000

    Introducing caching of build trees
    
    `cache-build-tree` var has been added to the project.conf defaults (defaulting to False)
    When cache-build-tree is set to True, the contents of the sandbox build directory is copied to the cache
    The cache key dict has been updated to reflect the status of `cache-build-tree`
---
 buildstream/_frontend/cli.py        |  5 ++++-
 buildstream/_project.py             |  3 ++-
 buildstream/data/projectconfig.yaml |  3 +++
 buildstream/element.py              | 20 +++++++++++++++++++-
 4 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 41736ac..6c9adb7 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -655,10 +655,13 @@ def workspace_close(app, remove_dir, all_, elements):
               help="Track and fetch the latest source before resetting")
 @click.option('--all', '-a', 'all_', default=False, is_flag=True,
               help="Reset all open workspaces")
+@click.option('--no-cache', 'no_cache', default=False, is_flag=True,
+              help="Use Cached build tree if available")
 @click.argument('elements', nargs=-1,
                 type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
-def workspace_reset(app, track_, all_, elements):
+def workspace_reset(app, track_, all_, element, no_cache):
+                type=click.Path(dir_okay=False, readable=True))
     """Reset a workspace to its original state"""
 
     # Check that the workspaces in question exist
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 12074ab..fec39ee 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -249,7 +249,8 @@ class Project():
             'aliases', 'name',
             'artifacts', 'options',
             'fail-on-overlap', 'shell',
-            'ref-storage', 'sandbox'
+            'ref-storage', 'sandbox',
+            'cache-build-tree'
         ])
 
         # The project name, element path and option declarations
diff --git a/buildstream/data/projectconfig.yaml b/buildstream/data/projectconfig.yaml
index b4ad2dc..f512f1d 100644
--- a/buildstream/data/projectconfig.yaml
+++ b/buildstream/data/projectconfig.yaml
@@ -33,6 +33,9 @@ variables:
   element-name: ""
   project-name: ""
 
+  # Whether or not to cache build trees
+  cache-build-tree: True
+
   # Path configuration, to be used in build instructions.
   #
   prefix: "/usr"
diff --git a/buildstream/element.py b/buildstream/element.py
index c563b79..64a3f79 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -928,6 +928,17 @@ class Element(Plugin):
     def _get_redundant_source_refs(cls):
         return cls.__redundant_source_refs
 
+    # _build_tree_path():
+    #    
+    # Returns the path of the cached build tree if it exists
+    #    
+    def _build_tree_path(self):
+        build_tree_path = os.path.join(self.__extract()[0], 'buildtree')
+        if os.path.isdir(build_tree_path):
+            return build_tree_path
+        else:
+            return None 
+
     # _reset_load_state()
     #
     # This is called by Pipeline.cleanup() and is used to
@@ -1479,6 +1490,12 @@ class Element(Plugin):
                 # Hard link files from collect dir to files directory
                 utils.link_files(collectdir, filesdir)
 
+                # Copy build tree contents
+                if self.get_variable('cache-build-tree'):
+                    sandbox_build_dir = os.path.join(sandbox_root, self.get_variable('build-root'))
+                    if os.path.isdir(sandbox_build_dir):
+                        shutil.copytree(sandbox_build_dir, os.path.join(assembledir, 'buildtree'))
+
                 # Copy build log
                 if self.__log_path:
                     shutil.copyfile(self.__log_path, os.path.join(logsdir, 'build.log'))
@@ -1905,7 +1922,8 @@ class Element(Plugin):
                 'sources': [s._get_unique_key(workspace is None) for s in self.__sources],
                 'workspace': '' if workspace is None else workspace.get_key(),
                 'public': self.__public,
-                'cache': type(self.__artifacts).__name__
+                'cache': type(self.__artifacts).__name__,
+                'cache-build-tree': self.get_variable('cache-build-tree')
             }
 
         cache_key_dict = self.__cache_key_dict.copy()