You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:58:18 UTC
[buildstream] branch aevri/win32_temptext created (now 8d658f0)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch aevri/win32_temptext
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 8d658f0 utils: indent 'with' continuation line
This branch includes the following new commits:
new 736e417 _yaml.dump: allow pass-through of file handle
new 1ea866d utils: add _TempTextBuffer for avoiding tmp files
new d97f909 _artifact: use _TempTextBuffer instead of tmp file
new a86657c cascache: refactor, rm some unused exception vars
new 8cab8a1 cascache: add_object, assert path if link_directly
new 8d658f0 utils: indent 'with' continuation line
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 05/06: cascache: add_object,
assert path if link_directly
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/win32_temptext
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8cab8a1054207a79cb5f09b78234309f56a53220
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jun 11 10:12:14 2019 +0100
cascache: add_object, assert path if link_directly
---
src/buildstream/_cas/cascache.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 005fd98..d9832fe 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -381,6 +381,9 @@ class CASCache():
# Exactly one of the two parameters has to be specified
assert (path is None) != (buffer is None)
+ # If we're linking directly, then path must be specified.
+ assert (not link_directly) or (link_directly and path)
+
if digest is None:
digest = remote_execution_pb2.Digest()
[buildstream] 06/06: utils: indent 'with' continuation line
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/win32_temptext
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8d658f0714c607c94a250b14dcaf8ad8c28f5d9b
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jun 11 10:44:43 2019 +0100
utils: indent 'with' continuation line
---
src/buildstream/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index 79cbbde..b47bb47 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -1015,7 +1015,7 @@ def _tempnamedfile(suffix="", prefix="tmp", dir=None): # pylint: disable=redefi
temp.close()
with _signals.terminator(close_tempfile), \
- tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, dir=dir) as temp:
+ tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, dir=dir) as temp:
yield temp
[buildstream] 03/06: _artifact: use _TempTextBuffer instead of tmp
file
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/win32_temptext
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d97f9095597b05d669db3f81c5a6b51bca62392c
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Jun 6 16:28:10 2019 +0100
_artifact: use _TempTextBuffer instead of tmp file
When adding public data to CAS, save on file I/O by passing an in-memory
buffer to _cas.add_object() instead of a filename to read. This means
that we avoid reading back the file from disk when hashing it, we still
only write it to disk once.
This also side-steps a win32 compatability issue, the 'name' member of
tempfile.NamedTemporaryFile cannot be opened on Windows NT or later, as
per the Python docs:
https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
---
src/buildstream/_artifact.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index 90d25b4..b0db06d 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -29,7 +29,6 @@ artifact composite interaction away from Element class
"""
import os
-import tempfile
from ._protos.buildstream.v2.artifact_pb2 import Artifact as ArtifactProto
from . import _yaml
@@ -148,9 +147,9 @@ class Artifact():
size += filesvdir.get_size()
# Store public data
- with tempfile.NamedTemporaryFile(dir=self._tmpdir) as tmp:
- _yaml.dump(_yaml.node_sanitize(publicdata), tmp.name)
- public_data_digest = self._cas.add_object(path=tmp.name, link_directly=True)
+ with utils._TempTextBuffer() as tmp:
+ _yaml.dump(_yaml.node_sanitize(publicdata), tmp.stream)
+ public_data_digest = self._cas.add_object(buffer=tmp.get_bytes_copy())
artifact.public_data.CopyFrom(public_data_digest)
size += public_data_digest.size_bytes
[buildstream] 04/06: cascache: refactor,
rm some unused exception vars
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/win32_temptext
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a86657c277e574bffa3f7b03aa7028cf4c68eaf3
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jun 11 10:10:16 2019 +0100
cascache: refactor, rm some unused exception vars
---
src/buildstream/_cas/cascache.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index ad8013d..005fd98 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -260,7 +260,7 @@ class CASCache():
raise CASCacheError("Failed to pull ref {}: {}".format(ref, e)) from e
else:
return False
- except BlobNotFound as e:
+ except BlobNotFound:
return False
# pull_tree():
@@ -414,7 +414,7 @@ class CASCache():
os.makedirs(os.path.dirname(objpath), exist_ok=True)
os.link(tmp.name, objpath)
- except FileExistsError as e:
+ except FileExistsError:
# We can ignore the failed link() if the object is already in the repo.
pass
[buildstream] 02/06: utils: add _TempTextBuffer for avoiding tmp
files
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/win32_temptext
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1ea866dcfa6f8eb54c2fad85f2b9ec902c198274
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Jun 6 16:23:59 2019 +0100
utils: add _TempTextBuffer for avoiding tmp files
Add a convenience context manager for temporary text buffers, backed by
StringIO. This can be used instead of temporary files in some cases,
saving on file I/O and compatibility concerns.
---
src/buildstream/utils.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 49 insertions(+)
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index f509ce9..79cbbde 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -24,6 +24,8 @@ Utilities
import calendar
import errno
import hashlib
+import io
+import locale
import os
import re
import shutil
@@ -1278,6 +1280,53 @@ def _search_upward_for_files(directory, filenames):
directory = parent_dir
+# A context manager for temporary text buffers.
+#
+# This can be used instead of temporary files in some cases, saving on file I/O
+# and compatibility concerns.
+#
+# The 'stream' member can be used in a similar way to an open text file.
+#
+# The getbuffer() method returns a bytes object that may be used with e.g.
+# cas.add_object().
+#
+# Note that the format of bytes written in the buffer are the same as
+# `open("tmp", "w")`:
+#
+# - The line separator is `os.linesep`.
+# - The encoding is `locale.getpreferredencoding()`.
+#
+class _TempTextBuffer():
+
+ def __init__(self):
+ self.stream = None
+
+ def __enter__(self):
+ self.stream = io.StringIO(newline=os.linesep)
+ return self
+
+ def __exit__(self, *_):
+ self.stream.close()
+ self.stream = None
+
+ # get_bytes_copy()
+ #
+ # Returns a copy of the encoded bytes of the text written to the buffer.
+ #
+ # Returns:
+ # (bytes): The encoded bytes of the text written to this buffer.
+ #
+ def get_bytes_copy(self):
+ if self.stream is None:
+ raise Exception(
+ "Must be used as a context manager in a 'with' statement.")
+ strval = self.stream.getvalue()
+ bytestring = strval.encode(
+ locale.getpreferredencoding(do_setlocale=False)
+ )
+ return bytestring
+
+
# _deterministic_umask()
#
# Context managed to apply a umask to a section that may be affected by a users
[buildstream] 01/06: _yaml.dump: allow pass-through of file handle
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch aevri/win32_temptext
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 736e417a50f9f85f219f7b337af4f76bb19e3369
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Wed Jun 5 17:32:38 2019 +0100
_yaml.dump: allow pass-through of file handle
The underlying roundtrip_dump() allows for the 'file' parameter to be a
filename or a file handle. Offer this flexibility with dump() too.
Also copy the slighlty better documentation for the roundtrip_dump()
parameters to dump().
---
src/buildstream/_yaml.pyx | 12 ++++++++----
tests/artifactcache/config.py | 4 ++--
tests/artifactcache/junctions.py | 2 +-
tests/artifactcache/pull.py | 4 ++--
tests/artifactcache/push.py | 4 ++--
tests/cachekey/cachekey.py | 4 ++--
tests/frontend/remote-caches.py | 2 +-
tests/sourcecache/fetch.py | 6 +++---
tests/sourcecache/push.py | 6 +++---
9 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/src/buildstream/_yaml.pyx b/src/buildstream/_yaml.pyx
index 2122dea..d47316d 100644
--- a/src/buildstream/_yaml.pyx
+++ b/src/buildstream/_yaml.pyx
@@ -460,11 +460,15 @@ cpdef Node load_data(str data, int file_index=_SYNTHETIC_FILE_INDEX, str file_na
# to output something close to what you read in, consider using the
# `roundtrip_load` and `roundtrip_dump` function pair instead.
#
+# If `file` is a string, it is the filename to write to, if `file` has a
+# `write` method, it's treated as a stream, otherwise output is to stdout.
+#
# Args:
-# contents (any): Content to write out
-# filename (str): The (optional) file name to write out to
-def dump(object contents, str filename=None):
- roundtrip_dump(node_sanitize(contents), file=filename)
+# contents (Mapping or list): The content to write out as YAML.
+# file (any): The file to write to.
+#
+def dump(contents, file=None):
+ roundtrip_dump(node_sanitize(contents), file=file)
# node_get_provenance()
diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index 591cc0e..59cd4fb 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -101,11 +101,11 @@ def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user
project_config['name'] = 'test'
user_config_file = str(tmpdir.join('buildstream.conf'))
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
project_dir = tmpdir.mkdir('project')
project_config_file = str(project_dir.join('project.conf'))
- _yaml.dump(_yaml.node_sanitize(project_config), filename=project_config_file)
+ _yaml.dump(_yaml.node_sanitize(project_config), file=project_config_file)
context = Context()
context.load(config=user_config_file)
diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py
index 52d721b..98be62b 100644
--- a/tests/artifactcache/junctions.py
+++ b/tests/artifactcache/junctions.py
@@ -35,7 +35,7 @@ def project_set_artifacts(project, url):
'url': url,
'push': True
})
- _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
+ _yaml.dump(_yaml.node_sanitize(project_config), file=project_conf_file)
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index d5f2c1f..eb36820 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -73,7 +73,7 @@ def test_pull(cli, tmpdir, datafiles):
}
# Write down the user configuration file
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
# Ensure CLI calls will use it
cli.configure(user_config)
@@ -182,7 +182,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
}
# Write down the user configuration file
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
# Ensure CLI calls will use it
cli.configure(user_config)
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 1cf46a4..fb9ddbb 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -66,7 +66,7 @@ def test_push(cli, tmpdir, datafiles):
}
# Write down the user configuration file
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
# Fake minimal context
context = Context()
@@ -164,7 +164,7 @@ def test_push_message(tmpdir, datafiles):
}
# Write down the user configuration file
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
queue = multiprocessing.Queue()
# Use subprocess to avoid creation of gRPC threads in main BuildStream process
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index 7c5d90d..1591743 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -198,11 +198,11 @@ def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings,
project_dir = tmpdir.mkdir(project_name)
project_config_file = str(project_dir.join('project.conf'))
- _yaml.dump(_yaml.node_sanitize(config), filename=project_config_file)
+ _yaml.dump(_yaml.node_sanitize(config), file=project_config_file)
elem_dir = project_dir.mkdir('elements')
element_file = str(elem_dir.join('stack.bst'))
- _yaml.dump({'kind': 'stack'}, filename=element_file)
+ _yaml.dump({'kind': 'stack'}, file=element_file)
result = cli.run(project=str(project_dir), args=[
'show',
diff --git a/tests/frontend/remote-caches.py b/tests/frontend/remote-caches.py
index 089cf96..2e8b270 100644
--- a/tests/frontend/remote-caches.py
+++ b/tests/frontend/remote-caches.py
@@ -57,7 +57,7 @@ def test_source_artifact_caches(cli, tmpdir, datafiles):
},
'cachedir': cachedir
}
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
cli.configure(user_config)
create_element_size('repo.bst', project_dir, element_path, [], 10000)
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index 899e162..e6dcf07 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -56,7 +56,7 @@ def test_source_fetch(cli, tmpdir, datafiles):
},
'cachedir': cache_dir,
}
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
cli.configure(user_config)
repo = create_repo('git', str(tmpdir))
@@ -131,7 +131,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
},
'cachedir': cache_dir,
}
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
cli.configure(user_config)
repo = create_repo('git', str(tmpdir))
@@ -188,7 +188,7 @@ def test_pull_fail(cli, tmpdir, datafiles):
},
'cachedir': cache_dir,
}
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
cli.configure(user_config)
repo = create_repo('git', str(tmpdir))
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 065047b..e2ff612 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -56,7 +56,7 @@ def test_source_push(cli, tmpdir, datafiles):
},
'cachedir': cache_dir,
}
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
cli.configure(user_config)
repo = create_repo('git', str(tmpdir))
@@ -116,7 +116,7 @@ def test_push_pull(cli, datafiles, tmpdir):
},
'cachedir': cache_dir,
}
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
cli.configure(user_config)
# create repo to pull from
@@ -162,7 +162,7 @@ def test_push_fail(cli, tmpdir, datafiles):
},
'cachedir': cache_dir,
}
- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
+ _yaml.dump(_yaml.node_sanitize(user_config), file=user_config_file)
cli.configure(user_config)
# create repo to pull from