You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:58:42 UTC
[buildstream] 06/26: _casbaseddirectory: Add properties to
IndexEntry
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch traveltissues/mr4
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a45c76586c590493cbe80de5e94661ec750f9eb3
Author: Darius Makovsky <tr...@protonmail.com>
AuthorDate: Mon Dec 2 14:41:30 2019 +0000
_casbaseddirectory: Add properties to IndexEntry
supports properties in `_partial_import_cas_into_cas`
_casbaseddirectory: initialise entrynode properties
Copy node properties from/to IndexEntry
_casbaseddirectory: sort properties before copying to filenode
---
src/buildstream/storage/_casbaseddirectory.py | 37 +++++++++++++++++++++-----
src/buildstream/storage/_filebaseddirectory.py | 2 +-
src/buildstream/storage/directory.py | 2 +-
3 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py
index bfa3f82..377a461 100644
--- a/src/buildstream/storage/_casbaseddirectory.py
+++ b/src/buildstream/storage/_casbaseddirectory.py
@@ -29,13 +29,14 @@ See also: :ref:`sandboxing`.
import os
import stat
+import copy
import tarfile as tarfilelib
from io import StringIO
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from .directory import Directory, VirtualDirectoryError, _FileType
from ._filebaseddirectory import FileBasedDirectory
-from ..utils import FileListResult, BST_ARBITRARY_TIMESTAMP
+from ..utils import FileListResult, BST_ARBITRARY_TIMESTAMP, _get_file_mtimestamp
class IndexEntry:
@@ -50,7 +51,8 @@ class IndexEntry:
target=None,
is_executable=False,
buildstream_object=None,
- modified=False
+ modified=False,
+ node_properties=None
):
self.name = name
self.type = entrytype
@@ -59,6 +61,7 @@ class IndexEntry:
self.is_executable = is_executable
self.buildstream_object = buildstream_object
self.modified = modified
+ self.node_properties = copy.deepcopy(node_properties)
def get_directory(self, parent):
if not self.buildstream_object:
@@ -126,7 +129,11 @@ class CasBasedDirectory(Directory):
self.index[entry.name] = IndexEntry(entry.name, _FileType.DIRECTORY, digest=entry.digest)
for entry in pb2_directory.files:
self.index[entry.name] = IndexEntry(
- entry.name, _FileType.REGULAR_FILE, digest=entry.digest, is_executable=entry.is_executable
+ entry.name,
+ _FileType.REGULAR_FILE,
+ digest=entry.digest,
+ is_executable=entry.is_executable,
+ node_properties=entry.node_properties,
)
for entry in pb2_directory.symlinks:
self.index[entry.name] = IndexEntry(entry.name, _FileType.SYMLINK, target=entry.target)
@@ -150,11 +157,18 @@ class CasBasedDirectory(Directory):
return newdir
- def _add_file(self, basename, filename, modified=False, can_link=False):
+ def _add_file(self, basename, filename, modified=False, can_link=False, properties=None):
entry = IndexEntry(filename, _FileType.REGULAR_FILE, modified=modified or filename in self.index)
path = os.path.join(basename, filename)
entry.digest = self.cas_cache.add_object(path=path, link_directly=can_link)
entry.is_executable = os.access(path, os.X_OK)
+ properties = properties or []
+ entry.node_properties = []
+ if "MTime" in properties:
+ node_property = remote_execution_pb2.NodeProperty()
+ node_property.name = "MTime"
+ node_property.value = _get_file_mtimestamp(path)
+ entry.node_properties.append(node_property)
self.index[filename] = entry
self.__invalidate_digest()
@@ -333,6 +347,7 @@ class CasBasedDirectory(Directory):
digest=entry.digest,
is_executable=entry.is_executable,
modified=True,
+ node_properties=entry.node_properties,
)
self.__invalidate_digest()
else:
@@ -341,7 +356,14 @@ class CasBasedDirectory(Directory):
result.files_written.append(relative_pathname)
def import_files(
- self, external_pathspec, *, filter_callback=None, report_written=True, update_mtime=False, can_link=False
+ self,
+ external_pathspec,
+ *,
+ filter_callback=None,
+ report_written=True,
+ update_mtime=False,
+ can_link=False,
+ properties=None
):
""" See superclass Directory for arguments """
@@ -368,13 +390,14 @@ class CasBasedDirectory(Directory):
return result
- def import_single_file(self, external_pathspec):
+ def import_single_file(self, external_pathspec, properties=None):
result = FileListResult()
if self._check_replacement(os.path.basename(external_pathspec), os.path.dirname(external_pathspec), result):
self._add_file(
os.path.dirname(external_pathspec),
os.path.basename(external_pathspec),
modified=os.path.basename(external_pathspec) in result.overwritten,
+ properties=properties,
)
result.files_written.append(external_pathspec)
return result
@@ -639,6 +662,8 @@ class CasBasedDirectory(Directory):
filenode.name = name
filenode.digest.CopyFrom(entry.digest)
filenode.is_executable = entry.is_executable
+ if entry.node_properties:
+ filenode.node_properties.extend(copy.deepcopy(sorted(entry.node_properties)))
elif entry.type == _FileType.SYMLINK:
symlinknode = pb2_directory.symlinks.add()
symlinknode.name = name
diff --git a/src/buildstream/storage/_filebaseddirectory.py b/src/buildstream/storage/_filebaseddirectory.py
index 95d113e..d01b5ce 100644
--- a/src/buildstream/storage/_filebaseddirectory.py
+++ b/src/buildstream/storage/_filebaseddirectory.py
@@ -130,7 +130,7 @@ class FileBasedDirectory(Directory):
os.utime(os.path.join(self.external_directory, f), times=(cur_time, cur_time))
return import_result
- def import_single_file(self, external_pathspec):
+ def import_single_file(self, external_pathspec, properties=None):
dstpath = os.path.join(self.external_directory, os.path.basename(external_pathspec))
result = FileListResult()
if os.path.exists(dstpath):
diff --git a/src/buildstream/storage/directory.py b/src/buildstream/storage/directory.py
index 4cec772..92388b1 100644
--- a/src/buildstream/storage/directory.py
+++ b/src/buildstream/storage/directory.py
@@ -114,7 +114,7 @@ class Directory:
raise NotImplementedError()
- def import_single_file(self, external_pathspec):
+ def import_single_file(self, external_pathspec, properties=None):
"""Imports a single file from an external path"""
raise NotImplementedError()