You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:18:33 UTC
[buildstream] branch ctolentino/test created (now 5b0a1bb)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch ctolentino/test
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 5b0a1bb Force fail
This branch includes the following new commits:
new f4da999 casdprocessmanager.py: Relax timeout for establishing buildbox-casd connection
new 3f4c935 casdprocessmanager.py: Check if buildbox-casd process is alive while waiting for connection
new e8f9efb Create timed activity for waiting on buildbox-casd
new 5b0a1bb Force fail
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/04: casdprocessmanager.py: Relax timeout for
establishing buildbox-casd connection
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch ctolentino/test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f4da9998de28f25d49ca549da789adf0352e0bbe
Author: ctolentino8 <ct...@bloomberg.net>
AuthorDate: Thu Apr 9 15:25:40 2020 +0100
casdprocessmanager.py: Relax timeout for establishing buildbox-casd connection
---
src/buildstream/_cas/casdprocessmanager.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py
index 4c9d802..8948869 100644
--- a/src/buildstream/_cas/casdprocessmanager.py
+++ b/src/buildstream/_cas/casdprocessmanager.py
@@ -37,6 +37,7 @@ from .._exceptions import CASCacheError
from .._message import Message, MessageType
_CASD_MAX_LOGFILES = 10
+_CASD_TIMEOUT = 300 # in seconds
# CASDProcessManager
@@ -240,8 +241,8 @@ class CASDChannel:
while not os.path.exists(self._socket_path):
# casd is not ready yet, try again after a 10ms delay,
- # but don't wait for more than 15s
- if time.time() > self._start_time + 15:
+ # but don't wait for more than specified timeout period
+ if time.time() > self._start_time + _CASD_TIMEOUT:
raise CASCacheError("Timed out waiting for buildbox-casd to become ready")
time.sleep(0.01)
[buildstream] 04/04: Force fail
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch ctolentino/test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5b0a1bbe90c75aab745e09a39e523baa2c44a808
Author: ctolentino8 <ct...@bloomberg.net>
AuthorDate: Mon May 4 11:10:36 2020 +0000
Force fail
---
tests/integration/autotools.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index d1ab82e..ecae307 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -27,6 +27,8 @@ def test_autotools_build(cli, datafiles):
result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
+ assert False
+
result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
[buildstream] 03/04: Create timed activity for waiting on
buildbox-casd
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch ctolentino/test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e8f9efb272b66691969bb858f409d7618927b408
Author: ctolentino8 <ct...@bloomberg.net>
AuthorDate: Thu Apr 9 15:47:53 2020 +0100
Create timed activity for waiting on buildbox-casd
---
src/buildstream/_cas/cascache.py | 5 +--
src/buildstream/_cas/casdprocessmanager.py | 52 +++++++++++++++++++-----------
src/buildstream/_context.py | 1 +
3 files changed, 38 insertions(+), 20 deletions(-)
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 74912c4..795e04b 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -73,7 +73,8 @@ class CASCache:
cache_quota=None,
protect_session_blobs=True,
log_level=CASLogLevel.WARNING,
- log_directory=None
+ log_directory=None,
+ messenger=None,
):
self.casdir = os.path.join(path, "cas")
self.tmpdir = os.path.join(path, "tmp")
@@ -91,7 +92,7 @@ class CASCache:
path, log_dir, log_level, cache_quota, protect_session_blobs
)
- self._casd_channel = self._casd_process_manager.create_channel()
+ self._casd_channel = self._casd_process_manager.create_channel(messenger)
self._cache_usage_monitor = _CASCacheUsageMonitor(self._casd_channel)
def __getstate__(self):
diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py
index 5ba192d..f1c3826 100644
--- a/src/buildstream/_cas/casdprocessmanager.py
+++ b/src/buildstream/_cas/casdprocessmanager.py
@@ -223,12 +223,12 @@ class CASDProcessManager:
# Return a CASDChannel, note that the actual connection is not necessarily
# established until it is needed.
#
- def create_channel(self):
- return CASDChannel(self._socket_path, self._connection_string, self._start_time, self.process.pid)
+ def create_channel(self, messenger=None):
+ return CASDChannel(self._socket_path, self._connection_string, self._start_time, self.process.pid, messenger)
class CASDChannel:
- def __init__(self, socket_path, connection_string, start_time, casd_pid):
+ def __init__(self, socket_path, connection_string, start_time, casd_pid, messenger):
self._socket_path = socket_path
self._connection_string = connection_string
self._start_time = start_time
@@ -237,25 +237,41 @@ class CASDChannel:
self._casd_cas = None
self._local_cas = None
self._casd_pid = casd_pid
+ self._messenger = messenger
+
+ def __getstate__(self):
+ state = self.__dict__.copy()
+
+ # Messenger is not pickle-able
+ assert "_messenger" in state
+ state["_messenger"] = None
+
+ return state
def _establish_connection(self):
assert self._casd_channel is None
- while not os.path.exists(self._socket_path):
- # casd is not ready yet, try again after a 10ms delay,
- # but don't wait for more than specified timeout period
- if time.time() > self._start_time + _CASD_TIMEOUT:
- raise CASCacheError("Timed out waiting for buildbox-casd to become ready")
-
- # check that process is still alive
- try:
- proc = psutil.Process(self._casd_pid)
- if not proc.is_running():
- raise CASCacheError(f"buildbox-casd process died before connection could be established")
- except psutil.NoSuchProcess:
- raise CASCacheError("buildbox-casd process died before connection could be established")
-
- time.sleep(0.01)
+ if self._messenger and not os.path.exists(self._socket_path):
+ cm = self._messenger.timed_activity("Waiting for buildbox-casd")
+ else:
+ cm = contextlib.suppress()
+
+ with cm:
+ while not os.path.exists(self._socket_path):
+ # casd is not ready yet, try again after a 10ms delay,
+ # but don't wait for more than specified timeout period
+ if time.time() > self._start_time + _CASD_TIMEOUT:
+ raise CASCacheError("Timed out waiting for buildbox-casd to become ready")
+
+ # check that process is still alive
+ try:
+ proc = psutil.Process(self._casd_pid)
+ if not proc.is_running():
+ raise CASCacheError(f"buildbox-casd process died before connection could be established")
+ except psutil.NoSuchProcess:
+ raise CASCacheError("buildbox-casd process died before connection could be established")
+
+ time.sleep(0.01)
self._casd_channel = grpc.insecure_channel(self._connection_string)
self._bytestream = bytestream_pb2_grpc.ByteStreamStub(self._casd_channel)
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 090b3e0..38dbde6 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -539,6 +539,7 @@ class Context:
cache_quota=self.config_cache_quota,
log_level=log_level,
log_directory=self.logdir,
+ messenger=self.messenger,
)
return self._cascache
[buildstream] 02/04: casdprocessmanager.py: Check if buildbox-casd
process is alive while waiting for connection
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch ctolentino/test
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3f4c935778f777e1936b2af53fb499bc65c0a6d5
Author: ctolentino8 <ct...@bloomberg.net>
AuthorDate: Thu Apr 9 15:28:39 2020 +0100
casdprocessmanager.py: Check if buildbox-casd process is alive while waiting for connection
---
src/buildstream/_cas/casdprocessmanager.py | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py
index 8948869..5ba192d 100644
--- a/src/buildstream/_cas/casdprocessmanager.py
+++ b/src/buildstream/_cas/casdprocessmanager.py
@@ -25,6 +25,7 @@ import stat
import subprocess
import tempfile
import time
+import psutil
import grpc
@@ -223,11 +224,11 @@ class CASDProcessManager:
# established until it is needed.
#
def create_channel(self):
- return CASDChannel(self._socket_path, self._connection_string, self._start_time)
+ return CASDChannel(self._socket_path, self._connection_string, self._start_time, self.process.pid)
class CASDChannel:
- def __init__(self, socket_path, connection_string, start_time):
+ def __init__(self, socket_path, connection_string, start_time, casd_pid):
self._socket_path = socket_path
self._connection_string = connection_string
self._start_time = start_time
@@ -235,6 +236,7 @@ class CASDChannel:
self._bytestream = None
self._casd_cas = None
self._local_cas = None
+ self._casd_pid = casd_pid
def _establish_connection(self):
assert self._casd_channel is None
@@ -245,6 +247,14 @@ class CASDChannel:
if time.time() > self._start_time + _CASD_TIMEOUT:
raise CASCacheError("Timed out waiting for buildbox-casd to become ready")
+ # check that process is still alive
+ try:
+ proc = psutil.Process(self._casd_pid)
+ if not proc.is_running():
+ raise CASCacheError(f"buildbox-casd process died before connection could be established")
+ except psutil.NoSuchProcess:
+ raise CASCacheError("buildbox-casd process died before connection could be established")
+
time.sleep(0.01)
self._casd_channel = grpc.insecure_channel(self._connection_string)