You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:22:36 UTC
[buildstream] branch aevri/win32_minimal created (now 4d5ff56)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 4d5ff56 TEMP: job: print detail on exception
This branch includes the following new commits:
new 679639c _scheduler/jobs/job.py: sort imports
new 887522d cli.py: no fcntl on Windows
new 5a13253 win32: _platform/win32: add support for win32
new cdae0ea _platform: add does_support_signals() accessor
new 3c67037 _signals.suspendable:"outermost" -> "is_outermost"
new f955634 _signals.terminator: "outermost" -> "is_outermost"
new bf378bf _signals.suspendable: early-out on win32
new 1e08977 jobs/job: use named parameters for long list
new 4fe73d4 job: no redundant signal code on win32
new 6b3e1f6 scheduler: no redundant signal code on win32
new 98b2c87 element: platform independent _get_normal_name
new 23a9d36 WIP: win32: job: replace add_child_handler with thread
new c2a4ba3 WIP: win32: ImportElement - fix separators
new 7b5c871 WIP: win32: don't set owner of files
new 22bb841 WIP: win32: WINDOWS.md: repro instructions
new b05dfd4 WIP: job.py: no add_reader for files on win32
new 6d9d6a2 WIP: cascache: use inet socket on win32
new 8c60f3f WIP: is_fork_allowed logic for win32
new 7347768 MAYBE: _context: make cachedir an absolute path
new 20a47f0 LATER: sandboxnone: use initial SandboxNone
new 9493f28 TEMP: INFO log pickle length and time
new 3ece2d8 TEMP: bst-buildboxcasd-test
new 8cb9d86 TEMP: bst-job-replay
new c7c324f TEMP: ignore local windows files
new 4d5ff56 TEMP: job: print detail on exception
The 25 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 19/25: MAYBE: _context: make cachedir an absolute path
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 734776835de837616b22321fb2d8443b947ab48f
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 4 12:22:52 2019 +0100
MAYBE: _context: make cachedir an absolute path
---
src/buildstream/_context.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 74833e9..66a81fa 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -255,6 +255,9 @@ class Context():
if not os.path.isabs(path) and not (directory == 'workspacedir' and path == '.'):
raise LoadError("{} must be an absolute path".format(directory), LoadErrorReason.INVALID_DATA)
+ # Note that cachedir needs to be an absolute path, otherwise we'll crash later in atomic_write
+ # self.cachedir = os.path.abspath(self.cachedir)
+
# add directories not set by users
self.tmpdir = os.path.join(self.cachedir, 'tmp')
self.casdir = os.path.join(self.cachedir, 'cas')
[buildstream] 21/25: TEMP: INFO log pickle length and time
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9493f28b0e6aa834c5d93e40d633020016dffe9b
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jul 16 17:15:24 2019 +0100
TEMP: INFO log pickle length and time
---
src/buildstream/_scheduler/jobs/job.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index c769d0f..4aa7a11 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -22,11 +22,13 @@
# System imports
import asyncio
import datetime
+import io
import multiprocessing
import os
import pickle
import signal
import sys
+import time
import traceback
# BuildStream toplevel imports
@@ -229,12 +231,21 @@ class Job():
)
if self._scheduler.context.platform.does_multiprocessing_start_require_pickling():
+ then = time.time()
pickled = pickle_child_job(
child_job, self._scheduler.context.get_projects())
+ now = time.time()
+ pickled.seek(0, io.SEEK_END)
+ self.message(MessageType.INFO, "pickled len: {:,}".format(pickled.tell()))
+ self.message(MessageType.INFO, "pickle time: {}s".format(round(now - then, 2)))
+ pickled.seek(0)
+ then = time.time()
self._process = Process(
target=_do_pickled_child_job,
args=[pickled, self._queue],
)
+ now = time.time()
+ self.message(MessageType.INFO, "make process: {}s".format(round(now - then, 2)))
else:
self._process = Process(
target=child_job.child_action,
@@ -245,11 +256,14 @@ class Job():
# the child process does not inherit the parent's state, but the main
# process will be notified of any signal after we launch the child.
#
+ then = time.time()
if does_platform_support_signals:
with _signals.blocked([signal.SIGINT, signal.SIGTSTP, signal.SIGTERM], ignore=False):
self._process.start()
else:
self._process.start()
+ now = time.time()
+ self.message(MessageType.INFO, "start process: {}s".format(round(now - then, 2)))
# Wait for the child task to complete.
#
[buildstream] 18/25: WIP: is_fork_allowed logic for win32
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8c60f3f63bd41926befd670ade49e0792dba2d2f
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Oct 3 10:33:56 2019 +0100
WIP: is_fork_allowed logic for win32
---
src/buildstream/_scheduler/scheduler.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py
index 6385b61..f1b861e 100644
--- a/src/buildstream/_scheduler/scheduler.py
+++ b/src/buildstream/_scheduler/scheduler.py
@@ -161,7 +161,7 @@ class Scheduler():
#
def run(self, queues):
- assert self.context.is_fork_allowed()
+ # assert self.context.is_fork_allowed()
# Hold on to the queues to process
self.queues = queues
[buildstream] 14/25: WIP: win32: don't set owner of files
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7b5c8717d64b8daf6f128cd118abadff4392af6d
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Wed Apr 17 16:33:44 2019 +0100
WIP: win32: don't set owner of files
---
src/buildstream/element.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index bbb0459..1df70bb 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1525,7 +1525,7 @@ class Element(Plugin):
# Ensure deterministic mtime of sources at build time
vdirectory.set_deterministic_mtime()
# Ensure deterministic owners of sources at build time
- vdirectory.set_deterministic_user()
+ #vdirectory.set_deterministic_user()
# _set_required():
#
[buildstream] 13/25: WIP: win32: ImportElement - fix separators
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c2a4ba35ce71dba56a36cc8fd246f1bcca908791
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Apr 2 13:36:29 2019 +0100
WIP: win32: ImportElement - fix separators
---
src/buildstream/plugins/elements/import.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py
index 9568bd0..103dafd 100644
--- a/src/buildstream/plugins/elements/import.py
+++ b/src/buildstream/plugins/elements/import.py
@@ -81,10 +81,10 @@ class ImportElement(Element):
outputdir = rootdir.descend('output', create=True)
# The directory to grab
- inputdir = inputdir.descend(*self.source.strip(os.sep).split(os.sep))
+ inputdir = inputdir.descend(*self.source.strip('/').split('/'))
# The output target directory
- outputdir = outputdir.descend(*self.target.strip(os.sep).split(os.sep), create=True)
+ outputdir = outputdir.descend(*self.target.strip('/').split('/'), create=True)
if inputdir.is_empty():
raise ElementError("{}: No files were found inside directory '{}'"
@@ -94,7 +94,7 @@ class ImportElement(Element):
outputdir.import_files(inputdir)
# And we're done
- return '/output'
+ return os.sep + 'output'
def generate_script(self):
build_root = self.get_variable('build-root')
[buildstream] 12/25: WIP: win32: job: replace add_child_handler
with thread
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 23a9d3684baa3c4f3111e46c9655c3242c1cceb1
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Apr 2 13:26:08 2019 +0100
WIP: win32: job: replace add_child_handler with thread
---
src/buildstream/_scheduler/jobs/job.py | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 46bb6bf..513af70 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -47,6 +47,21 @@ class _ReturnCode(FastEnum):
SKIPPED = 3
+def _call_on_waitpid_threadfun(running_loop, process, callback):
+ process.join()
+ running_loop.call_soon_threadsafe(callback, process.pid, process.exitcode)
+
+
+def call_on_waitpid(running_loop, pid, callback):
+ import threading
+ t = threading.Thread(
+ target=_call_on_waitpid_threadfun,
+ args=(running_loop, pid, callback)
+ )
+ t.start()
+ return t
+
+
# JobStatus:
#
# The job completion status, passed back through the
@@ -258,8 +273,7 @@ class Job():
# an event loop callback. Otherwise, if the job completes too fast, then
# the callback is called immediately.
#
- self._watcher = asyncio.get_child_watcher()
- self._watcher.add_child_handler(self._process.pid, self._parent_child_completed)
+ self._watcher = call_on_waitpid(self._scheduler.loop, self._process, self._parent_child_completed)
# terminate()
#
[buildstream] 22/25: TEMP: bst-buildboxcasd-test
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3ece2d8b0babae2e9ffd1a91f873b0a085f79880
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Aug 30 09:56:57 2019 +0100
TEMP: bst-buildboxcasd-test
---
setup.py | 3 +-
src/buildstream/buildboxcasdtest.py | 92 +++++++++++++++++++++++++++++++++++++
2 files changed, 94 insertions(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 19779fb..04038c6 100755
--- a/setup.py
+++ b/setup.py
@@ -156,7 +156,8 @@ bst_install_entry_points = {
if not os.environ.get('BST_ARTIFACTS_ONLY', ''):
check_for_bwrap()
bst_install_entry_points['console_scripts'] += [
- 'bst = buildstream._frontend:cli'
+ 'bst = buildstream._frontend:cli',
+ 'bst-buildboxcasd-test = buildstream.buildboxcasdtest:cli',
]
#####################################################
diff --git a/src/buildstream/buildboxcasdtest.py b/src/buildstream/buildboxcasdtest.py
new file mode 100644
index 0000000..9bc617e
--- /dev/null
+++ b/src/buildstream/buildboxcasdtest.py
@@ -0,0 +1,92 @@
+import os
+import pathlib
+import shutil
+import subprocess
+import tempfile
+import time
+
+import click
+import grpc
+
+# from ._protos.google.rpc import code_pb2
+from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
+from ._protos.build.buildgrid import local_cas_pb2, local_cas_pb2_grpc
+# from ._protos.buildstream.v2 import buildstream_pb2
+
+from . import utils
+
+
+@click.command(name='bst-buildboxcasd-test', short_help="Test buildboxcasd")
+@click.option('--no-server', is_flag=True, default=False, help="Don't start a casd server.")
+def cli(no_server):
+ path = os.path.abspath('./castemp')
+ os.makedirs(path, exist_ok=True)
+
+ # Place socket in global/user temporary directory to avoid hitting
+ # the socket path length limit.
+ # casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream')
+ # casd_socket_path = os.path.join(casd_socket_tempdir, 'casd.sock')
+ # casd_conn_str = f'unix:{casd_socket_path}'
+ casd_conn_str = f'localhost:9000'
+
+ if not no_server:
+ casd_args = [utils.get_host_tool('buildbox-casd')]
+ casd_args.append('--bind=' + casd_conn_str)
+ casd_args.append('--verbose')
+
+ casd_args.append(path)
+ casd_process = subprocess.Popen(casd_args, cwd=path)
+ casd_start_time = time.time()
+
+ # time.sleep(3)
+ local_cas = _get_local_cas(casd_start_time, casd_conn_str)
+ # print(local_cas)
+
+ path_to_add = pathlib.Path('file_to_add')
+ path_to_add.write_text("Hello!")
+
+ for _ in range(20):
+ request = local_cas_pb2.CaptureFilesRequest()
+ request.path.append(os.path.abspath(str(path_to_add)))
+ response = local_cas.CaptureFiles(request)
+ print('.', end='', flush=True)
+ # print(response)
+
+ time.sleep(3)
+
+ if not no_server:
+ casd_process.terminate()
+ try:
+ # Don't print anything if buildbox-casd terminates quickly
+ casd_process.wait(timeout=0.5)
+ except subprocess.TimeoutExpired:
+ try:
+ casd_process.wait(timeout=15)
+ except subprocess.TimeoutExpired:
+ casd_process.kill()
+ casd_process.wait(timeout=15)
+ casd_process = None
+
+ # shutil.rmtree(casd_socket_tempdir)
+
+
+def _get_local_cas(casd_start_time, casd_conn_str):
+ casd_channel = grpc.insecure_channel(casd_conn_str)
+ local_cas = local_cas_pb2_grpc.LocalContentAddressableStorageStub(casd_channel)
+
+ # Call GetCapabilities() to establish connection to casd
+ capabilities = remote_execution_pb2_grpc.CapabilitiesStub(casd_channel)
+ while True:
+ try:
+ capabilities.GetCapabilities(remote_execution_pb2.GetCapabilitiesRequest())
+ break
+ except grpc.RpcError as e:
+ print(e)
+ if e.code() == grpc.StatusCode.UNAVAILABLE:
+ if time.time() < casd_start_time + 2:
+ time.sleep(0.5)
+ continue
+
+ raise
+
+ return local_cas
[buildstream] 20/25: LATER: sandboxnone: use initial SandboxNone
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 20a47f024620f57a9ef0415e290911034e0ac81b
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jul 16 14:20:21 2019 +0100
LATER: sandboxnone: use initial SandboxNone
---
src/buildstream/_platform/darwin.py | 4 +-
src/buildstream/_platform/win32.py | 4 +-
src/buildstream/sandbox/__init__.py | 1 +
src/buildstream/sandbox/_sandboxnone.py | 68 +++++++++++++++++++++++++++++++++
4 files changed, 73 insertions(+), 4 deletions(-)
diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py
index f235353..6c06cff 100644
--- a/src/buildstream/_platform/darwin.py
+++ b/src/buildstream/_platform/darwin.py
@@ -18,7 +18,7 @@
import os
import resource
-from ..sandbox import SandboxDummy
+from ..sandbox import SandboxNone
from .platform import Platform
@@ -62,7 +62,7 @@ class Darwin(Platform):
kwargs['dummy_reason'] = \
"OSXFUSE is not supported and there are no supported sandbox " + \
"technologies for MacOS at this time"
- return SandboxDummy(*args, **kwargs)
+ return SandboxNone(*args, **kwargs)
def _setup_dummy_sandbox(self):
self.check_sandbox_config = Darwin._check_dummy_sandbox_config
diff --git a/src/buildstream/_platform/win32.py b/src/buildstream/_platform/win32.py
index 128f1ed..dd070ff 100644
--- a/src/buildstream/_platform/win32.py
+++ b/src/buildstream/_platform/win32.py
@@ -14,7 +14,7 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
-from ..sandbox import SandboxDummy
+from ..sandbox import SandboxNone
from .platform import Platform
@@ -48,7 +48,7 @@ class Win32(Platform):
@staticmethod
def _create_dummy_sandbox(*args, **kwargs):
kwargs['dummy_reason'] = "There are no supported sandbox technologies for Win32 at this time."
- return SandboxDummy(*args, **kwargs)
+ return SandboxNone(*args, **kwargs)
def _setup_dummy_sandbox(self):
self.check_sandbox_config = Win32._check_dummy_sandbox_config
diff --git a/src/buildstream/sandbox/__init__.py b/src/buildstream/sandbox/__init__.py
index 5966d19..6544348 100644
--- a/src/buildstream/sandbox/__init__.py
+++ b/src/buildstream/sandbox/__init__.py
@@ -20,3 +20,4 @@
from .sandbox import Sandbox, SandboxFlags, SandboxCommandError
from ._sandboxremote import SandboxRemote
from ._sandboxdummy import SandboxDummy
+from ._sandboxnone import SandboxNone
diff --git a/src/buildstream/sandbox/_sandboxnone.py b/src/buildstream/sandbox/_sandboxnone.py
new file mode 100644
index 0000000..f8a2bda
--- /dev/null
+++ b/src/buildstream/sandbox/_sandboxnone.py
@@ -0,0 +1,68 @@
+#
+# Copyright (C) 2019 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Angelos Evripiotis <je...@bloomberg.net>
+
+import pathlib
+import pprint
+import subprocess
+
+from .._exceptions import SandboxError
+from .sandbox import Sandbox
+
+
+class SandboxNone(Sandbox):
+
+ def __init__(self, *args, **kwargs):
+ # TODO: don't require a dict copy.
+ kwargs = kwargs.copy()
+ kwargs['allow_real_directory'] = True
+
+ super().__init__(*args, **kwargs)
+
+ uid = self._get_config().build_uid
+ gid = self._get_config().build_gid
+ if uid != 0 or gid != 0:
+ raise SandboxError("Chroot sandboxes cannot specify a non-root uid/gid "
+ "({},{} were supplied via config)".format(uid, gid))
+
+ self.mount_map = None
+
+ def _run(self, command, flags, *, cwd, env):
+
+ install_path = pathlib.Path(self.get_directory()) / 'buildstream-install'
+
+ env = env.copy()
+ env['BST_INSTALLPATH'] = str(install_path)
+
+ # TODO: figure out what to do with 'flags'.
+
+ # TODO: do this in a robust way.
+ if cwd.startswith("/"):
+ cwd = cwd[1:]
+
+ # pprint.pprint(env)
+
+ path = pathlib.Path(self.get_directory()) / cwd
+ print('run', command, 'in', path)
+ #result = subprocess.run(command, cwd=path, env=env)
+ result = subprocess.run(command, cwd=path)
+
+ # out = pathlib.Path(self.get_directory()) / 'buildstream-install'
+ # out.mkdir(exist_ok=True)
+
+ return result.returncode
[buildstream] 25/25: TEMP: job: print detail on exception
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4d5ff563ff9880360bf79f755c6752b21586cd2b
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 4 12:25:23 2019 +0100
TEMP: job: print detail on exception
Since we're not properly propagating errors back to the main process
yet, at least print problems out.
---
src/buildstream/_scheduler/jobs/job.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 4aa7a11..daa5233 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -850,6 +850,8 @@ class ChildJob():
elapsed = datetime.datetime.now() - starttime
detail = "An unhandled exception occured:\n\n{}".format(traceback.format_exc())
+ print(detail)
+
self.message(MessageType.BUG, self.action_name,
elapsed=elapsed, detail=detail,
logfile=filename)
[buildstream] 03/25: win32: _platform/win32: add support for win32
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5a132531f14e449d2183952adcdc2d81fd135bdd
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jul 16 14:17:54 2019 +0100
win32: _platform/win32: add support for win32
Copy the approach of 'Darwin' and provide a SandboxDummy.
This enables us to run 'bst workspace list' on Windows.
---
src/buildstream/_platform/platform.py | 4 +++
src/buildstream/_platform/win32.py | 56 +++++++++++++++++++++++++++++++++++
2 files changed, 60 insertions(+)
diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py
index 11c9217..af49b9e 100644
--- a/src/buildstream/_platform/platform.py
+++ b/src/buildstream/_platform/platform.py
@@ -98,6 +98,8 @@ class Platform():
backend = 'darwin'
elif sys.platform.startswith('linux'):
backend = 'linux'
+ elif sys.platform == 'win32':
+ backend = 'win32'
else:
backend = 'fallback'
@@ -105,6 +107,8 @@ class Platform():
from .linux import Linux as PlatformImpl # pylint: disable=cyclic-import
elif backend == 'darwin':
from .darwin import Darwin as PlatformImpl # pylint: disable=cyclic-import
+ elif backend == 'win32':
+ from .win32 import Win32 as PlatformImpl # pylint: disable=cyclic-import
elif backend == 'fallback':
from .fallback import Fallback as PlatformImpl # pylint: disable=cyclic-import
else:
diff --git a/src/buildstream/_platform/win32.py b/src/buildstream/_platform/win32.py
new file mode 100644
index 0000000..bb763cd
--- /dev/null
+++ b/src/buildstream/_platform/win32.py
@@ -0,0 +1,56 @@
+#
+# Copyright (C) 2019 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+from ..sandbox import SandboxDummy
+
+from .platform import Platform
+
+
+class Win32(Platform):
+
+ def maximize_open_file_limit(self):
+ # Note that on Windows, we don't have the 'resource' module to help us
+ # configure open file limits. By default we are allowed 512 open files,
+ # the `_setmaxstdio` C api may be used to increase this up to 8192.
+ #
+ # A grep for '_setmaxstdio' in the C python code base as at Python 3.8
+ # comes up empty, perhaps it will be added in later versions.
+ #
+ # 'psutil' provides an rlimit implementation that is only available on
+ # Linux, as of version 5.3. A grep of the psutil source for
+ # '_setmaxstdio' also comes up empty.
+ #
+ # To increase beyond the 512 limit on Windows, we will likely need to
+ # use Cython.
+ #
+ # For more information:
+ # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio
+ #
+ pass
+
+ @staticmethod
+ def _check_dummy_sandbox_config(config):
+ return True
+
+ @staticmethod
+ def _create_dummy_sandbox(*args, **kwargs):
+ kwargs['dummy_reason'] = "There are no supported sandbox technologies for Win32 at this time."
+ return SandboxDummy(*args, **kwargs)
+
+ def _setup_dummy_sandbox(self):
+ self.check_sandbox_config = Win32._check_dummy_sandbox_config
+ self.create_sandbox = Win32._create_dummy_sandbox
+ return True
\ No newline at end of file
[buildstream] 08/25: jobs/job: use named parameters for long list
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1e08977bacd4d722130dd3f6a608090a25c46b57
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 8 13:45:43 2019 +0100
jobs/job: use named parameters for long list
This invocation seems potentially error-prone as it is quite long, use
named parameters to make it a bit safer.
---
src/buildstream/_scheduler/jobs/job.py | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 940b7d2..7d901a3 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -201,14 +201,14 @@ class Job():
self._parent_start_listening()
child_job = self.create_child_job( # pylint: disable=assignment-from-no-return
- self.action_name,
- self._scheduler.context.messenger,
- self._scheduler.context.logdir,
- self._logfile,
- self._max_retries,
- self._tries,
- self._message_element_name,
- self._message_element_key
+ action_name=self.action_name,
+ messenger=self._scheduler.context.messenger,
+ logdir=self._scheduler.context.logdir,
+ logfile=self._logfile,
+ max_retries=self._max_retries,
+ tries=self._tries,
+ message_element_name=self._message_element_name,
+ message_element_key=self._message_element_key,
)
if self._scheduler.context.platform.does_multiprocessing_start_require_pickling():
[buildstream] 23/25: TEMP: bst-job-replay
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8cb9d860ac0971c20531d4ae0b5276cb3257a8e7
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jul 9 11:38:48 2019 +0100
TEMP: bst-job-replay
---
setup.py | 1 +
src/buildstream/_jobreplay.py | 13 +++++++++++++
src/buildstream/_scheduler/jobs/jobpickler.py | 4 ++++
3 files changed, 18 insertions(+)
diff --git a/setup.py b/setup.py
index 04038c6..2280b49 100755
--- a/setup.py
+++ b/setup.py
@@ -158,6 +158,7 @@ if not os.environ.get('BST_ARTIFACTS_ONLY', ''):
bst_install_entry_points['console_scripts'] += [
'bst = buildstream._frontend:cli',
'bst-buildboxcasd-test = buildstream.buildboxcasdtest:cli',
+ 'bst-job-replay = buildstream._jobreplay:cli',
]
#####################################################
diff --git a/src/buildstream/_jobreplay.py b/src/buildstream/_jobreplay.py
new file mode 100644
index 0000000..07782e4
--- /dev/null
+++ b/src/buildstream/_jobreplay.py
@@ -0,0 +1,13 @@
+import multiprocessing
+import pickle
+
+import click
+
+from ._scheduler.jobs.job import _do_pickled_child_job
+
+
+@click.command(name='bst-job-replay', short_help="Replay a bst job")
+@click.argument('replayfile', type=click.File("rb"))
+def cli(replayfile):
+ queue = multiprocessing.Queue()
+ _do_pickled_child_job(replayfile, queue)
\ No newline at end of file
diff --git a/src/buildstream/_scheduler/jobs/jobpickler.py b/src/buildstream/_scheduler/jobs/jobpickler.py
index 0edf88c..7de8404 100644
--- a/src/buildstream/_scheduler/jobs/jobpickler.py
+++ b/src/buildstream/_scheduler/jobs/jobpickler.py
@@ -94,6 +94,10 @@ def pickle_child_job(child_job, projects):
pickler.dump(child_job)
data.seek(0)
+ path = f"{child_job.action_name}_{child_job._message_element_name}_{child_job._message_element_key[1]}"
+ with open(path, "wb") as f:
+ f.write(data.getvalue())
+
return data
[buildstream] 01/25: _scheduler/jobs/job.py: sort imports
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 679639c7389753c00d2a18ed736955f938588cbb
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jul 9 11:44:48 2019 +0100
_scheduler/jobs/job.py: sort imports
---
src/buildstream/_scheduler/jobs/job.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 913e27e..940b7d2 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -20,14 +20,14 @@
# Tristan Maat <tr...@codethink.co.uk>
# System imports
+import asyncio
+import datetime
+import multiprocessing
import os
import pickle
-import sys
import signal
-import datetime
+import sys
import traceback
-import asyncio
-import multiprocessing
# BuildStream toplevel imports
from ..._exceptions import ImplError, BstError, set_last_task_error, SkipJob
[buildstream] 11/25: element: platform independent _get_normal_name
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 98b2c87e3d80433f133b49456596f10ec689bf3e
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Jun 25 14:23:43 2019 +0100
element: platform independent _get_normal_name
Note that windows has two path separators '/' and '\', handle both.
---
src/buildstream/element.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 4ebb17d..bbb0459 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -3369,7 +3369,10 @@ def _overlap_error_detail(f, forbidden_overlap_elements, elements):
# (str): The normalised element name
#
def _get_normal_name(element_name):
- return os.path.splitext(element_name.replace(os.sep, '-'))[0]
+ name_no_sep = element_name.replace(os.sep, '-')
+ if os.altsep is not None:
+ name_no_sep = name_no_sep.replace(os.altsep, '-')
+ return os.path.splitext(name_no_sep)[0]
# _compose_artifact_name():
[buildstream] 07/25: _signals.suspendable: early-out on win32
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bf378bf4a7fbf7433ef370b3d708ae89632e2b25
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 8 09:25:20 2019 +0100
_signals.suspendable: early-out on win32
---
src/buildstream/_signals.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/src/buildstream/_signals.py b/src/buildstream/_signals.py
index a7f32d6..a29cbdc 100644
--- a/src/buildstream/_signals.py
+++ b/src/buildstream/_signals.py
@@ -144,6 +144,12 @@ def suspend_handler(sig, frame):
#
@contextmanager
def suspendable(suspend_callback, resume_callback):
+ if sys.platform == 'win32':
+ # Win32 does not support SIGTSTP, at least up to Windows 10, so we
+ # won't be able to handle it here.
+ yield
+ return
+
global suspendable_stack # pylint: disable=global-statement
is_outermost = bool(not suspendable_stack)
[buildstream] 10/25: scheduler: no redundant signal code on win32
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6b3e1f65df0678185ff6f4afbf80f2858599637d
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 4 18:12:12 2019 +0100
scheduler: no redundant signal code on win32
There's no point disconnecting or blocking signals that don't exist on
e.g. win32. Some of these signal ids are undefined though, which causes
an AttributeError, so we want to avoid that.
---
src/buildstream/_scheduler/scheduler.py | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py
index d0a1895..6385b61 100644
--- a/src/buildstream/_scheduler/scheduler.py
+++ b/src/buildstream/_scheduler/scheduler.py
@@ -245,7 +245,8 @@ class Scheduler():
# Block this until we're finished terminating jobs,
# this will remain blocked forever.
- signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGINT])
+ if self.context.platform.does_support_signals():
+ signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGINT])
# jobs_suspended()
#
@@ -482,14 +483,16 @@ class Scheduler():
# Connects our signal handler event callbacks to the mainloop
#
def _connect_signals(self):
- self.loop.add_signal_handler(signal.SIGINT, self._interrupt_event)
- self.loop.add_signal_handler(signal.SIGTERM, self._terminate_event)
- self.loop.add_signal_handler(signal.SIGTSTP, self._suspend_event)
+ if self.context.platform.does_support_signals():
+ self.loop.add_signal_handler(signal.SIGINT, self._interrupt_event)
+ self.loop.add_signal_handler(signal.SIGTERM, self._terminate_event)
+ self.loop.add_signal_handler(signal.SIGTSTP, self._suspend_event)
def _disconnect_signals(self):
- self.loop.remove_signal_handler(signal.SIGINT)
- self.loop.remove_signal_handler(signal.SIGTSTP)
- self.loop.remove_signal_handler(signal.SIGTERM)
+ if self.context.platform.does_support_signals():
+ self.loop.remove_signal_handler(signal.SIGINT)
+ self.loop.remove_signal_handler(signal.SIGTSTP)
+ self.loop.remove_signal_handler(signal.SIGTERM)
def _terminate_jobs_real(self):
# 20 seconds is a long time, it can take a while and sometimes
[buildstream] 04/25: _platform: add does_support_signals() accessor
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cdae0ea7d17a713d914154b3bc6f7c4d1046a8e1
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 4 18:07:53 2019 +0100
_platform: add does_support_signals() accessor
We'll need this information for doing the right thing with signals on
Windows.
---
src/buildstream/_platform/platform.py | 12 ++++++++++++
src/buildstream/_platform/win32.py | 30 +++++++++++++++++++++++++++++-
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py
index af49b9e..b364ef8 100644
--- a/src/buildstream/_platform/platform.py
+++ b/src/buildstream/_platform/platform.py
@@ -190,6 +190,18 @@ class Platform():
# set to the platform default by `get_start_method`.
return multiprocessing.get_start_method() != 'fork'
+ # does_support_signals():
+ #
+ # Returns True if the platform has good support for signals, this will not
+ # be true for Windows.
+ #
+ # Returns:
+ # (bool): Whether signals are supported or not
+ #
+ def does_support_signals(self):
+ # Most platforms support signals, so the default is True.
+ return True
+
##################################################################
# Sandbox functions #
##################################################################
diff --git a/src/buildstream/_platform/win32.py b/src/buildstream/_platform/win32.py
index bb763cd..128f1ed 100644
--- a/src/buildstream/_platform/win32.py
+++ b/src/buildstream/_platform/win32.py
@@ -53,4 +53,32 @@ class Win32(Platform):
def _setup_dummy_sandbox(self):
self.check_sandbox_config = Win32._check_dummy_sandbox_config
self.create_sandbox = Win32._create_dummy_sandbox
- return True
\ No newline at end of file
+ return True
+
+ def does_support_signals(self):
+ # Windows does not have good support for signals, and we shouldn't
+ # handle them in the same way we do on UNIX.
+ #
+ # From the MSDN docs:
+ #
+ # > SIGINT is not supported for any Win32 application. When a CTRL+C
+ # > interrupt occurs, Win32 operating systems generate a new thread to
+ # > specifically handle that interrupt. This can cause a single-thread
+ # > application, such as one in UNIX, to become multithreaded and cause
+ # > unexpected behavior.
+ #
+ # > The SIGILL and SIGTERM signals are not generated under Windows.
+ # > They are included for ANSI compatibility. Therefore, you can set
+ # > signal handlers for these signals by using signal, and you can also
+ # > explicitly generate these signals by calling raise.
+ #
+ # The only other signals that are defined in signal.h on Windows are
+ # not relevant to us:
+ #
+ # - SIGABRT
+ # - SIGFPE
+ # - SIGSEGV
+ #
+ # https://docs.microsoft.com/en-gb/cpp/c-runtime-library/reference/signal
+ #
+ return False
[buildstream] 02/25: cli.py: no fcntl on Windows
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 887522d3dff2d3c5be30415696db6437ba2db6aa
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Aug 29 11:00:03 2019 +0100
cli.py: no fcntl on Windows
Work around the fact that we can't import 'fcntl' on Windows, and
confine the workaround to as small a scope as we can.
This enables us to run at least these commands on Windows:
bst help
bst init
We can't run any commands that require a Platform object though, which
is most commands.
---
src/buildstream/_frontend/cli.py | 29 +++++++++++++++++++++--------
1 file changed, 21 insertions(+), 8 deletions(-)
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 931f531..a0cf948 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -2,7 +2,6 @@ import multiprocessing
import os
import sys
from functools import partial
-import fcntl
import shutil
import click
@@ -187,6 +186,26 @@ def override_completions(orig_args, cmd, cmd_param, args, incomplete):
raise CompleteUnhandled()
+def validate_output_streams():
+ try:
+ import fcntl
+ except ImportError:
+ # When we move onto Python 3.6+, we'll probably want to check for
+ # ModuleNotFoundError instead, as that's more specifically the problem
+ # we're working around.
+ if sys.platform != 'win32':
+ raise
+ return
+
+ for stream in (sys.stdout, sys.stderr):
+ fileno = stream.fileno()
+ flags = fcntl.fcntl(fileno, fcntl.F_GETFL)
+ if flags & os.O_NONBLOCK:
+ click.echo("{} is currently set to O_NONBLOCK, try opening a new shell"
+ .format(stream.name), err=True)
+ sys.exit(-1)
+
+
def override_main(self, args=None, prog_name=None, complete_var=None,
standalone_mode=True, **extra):
@@ -211,13 +230,7 @@ def override_main(self, args=None, prog_name=None, complete_var=None,
# Check output file descriptor at earliest opportunity, to
# provide a reasonable error message instead of a stack trace
# in the case that it is blocking
- for stream in (sys.stdout, sys.stderr):
- fileno = stream.fileno()
- flags = fcntl.fcntl(fileno, fcntl.F_GETFL)
- if flags & os.O_NONBLOCK:
- click.echo("{} is currently set to O_NONBLOCK, try opening a new shell"
- .format(stream.name), err=True)
- sys.exit(-1)
+ validate_output_streams()
# We can only set the global multiprocessing start method once; for that
# reason we're advised to do it inside the entrypoint, where it is easy to
[buildstream] 09/25: job: no redundant signal code on win32
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4fe73d430e316a4c006f3df7e107fab731bc3332
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 4 18:11:02 2019 +0100
job: no redundant signal code on win32
There's no point blocking or handling signals that aren't supported on
e.g. win32. Some of these signal ids aren't defined on unsupported
platforms though, which would lead to an AttributeError at runtime.
Avoid that by not invoking the redundant code.
---
src/buildstream/_scheduler/jobs/job.py | 35 +++++++++++++++++++++-------------
1 file changed, 22 insertions(+), 13 deletions(-)
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 7d901a3..46bb6bf 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -200,6 +200,7 @@ class Job():
self._tries += 1
self._parent_start_listening()
+ does_platform_support_signals = self._scheduler.context.platform.does_support_signals()
child_job = self.create_child_job( # pylint: disable=assignment-from-no-return
action_name=self.action_name,
messenger=self._scheduler.context.messenger,
@@ -209,6 +210,7 @@ class Job():
tries=self._tries,
message_element_name=self._message_element_name,
message_element_key=self._message_element_key,
+ does_platform_support_signals=does_platform_support_signals,
)
if self._scheduler.context.platform.does_multiprocessing_start_require_pickling():
@@ -228,7 +230,10 @@ class Job():
# the child process does not inherit the parent's state, but the main
# process will be notified of any signal after we launch the child.
#
- with _signals.blocked([signal.SIGINT, signal.SIGTSTP, signal.SIGTERM], ignore=False):
+ if does_platform_support_signals:
+ with _signals.blocked([signal.SIGINT, signal.SIGTSTP, signal.SIGTERM], ignore=False):
+ self._process.start()
+ else:
self._process.start()
# Wait for the child task to complete.
@@ -631,7 +636,8 @@ class ChildJob():
def __init__(
self, action_name, messenger, logdir, logfile, max_retries, tries,
- message_element_name, message_element_key):
+ message_element_name, message_element_key,
+ does_platform_support_signals):
self.action_name = action_name
@@ -643,6 +649,8 @@ class ChildJob():
self._message_element_name = message_element_name
self._message_element_key = message_element_key
+ self._does_platform_support_signals = does_platform_support_signals
+
self._queue = None
# message():
@@ -732,17 +740,18 @@ class ChildJob():
#
def child_action(self, queue):
- # This avoids some SIGTSTP signals from grandchildren
- # getting propagated up to the master process
- os.setsid()
-
- # First set back to the default signal handlers for the signals
- # we handle, and then clear their blocked state.
- #
- signal_list = [signal.SIGTSTP, signal.SIGTERM]
- for sig in signal_list:
- signal.signal(sig, signal.SIG_DFL)
- signal.pthread_sigmask(signal.SIG_UNBLOCK, signal_list)
+ if (self._does_platform_support_signals):
+ # This avoids some SIGTSTP signals from grandchildren
+ # getting propagated up to the master process
+ os.setsid()
+
+ # First set back to the default signal handlers for the signals
+ # we handle, and then clear their blocked state.
+ #
+ signal_list = [signal.SIGTSTP, signal.SIGTERM]
+ for sig in signal_list:
+ signal.signal(sig, signal.SIG_DFL)
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, signal_list)
# Assign the queue we passed across the process boundaries
#
[buildstream] 17/25: WIP: cascache: use inet socket on win32
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6d9d6a2e6e9db567a2b6061d95c715d8d164c15d
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Aug 29 14:14:36 2019 +0100
WIP: cascache: use inet socket on win32
---
src/buildstream/_cas/cascache.py | 42 ++++++++++++++++++++++++++--------------
1 file changed, 28 insertions(+), 14 deletions(-)
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 6002adc..fe21251 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -69,11 +69,12 @@ class CASCache():
if casd:
# Place socket in global/user temporary directory to avoid hitting
# the socket path length limit.
- self._casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream')
- self._casd_socket_path = os.path.join(self._casd_socket_tempdir, 'casd.sock')
+ # self._casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream')
+ # self._casd_socket_path = os.path.join(self._casd_socket_tempdir, 'casd.sock')
+ self._casd_socket_connectstr = 'localhost:9000'
casd_args = [utils.get_host_tool('buildbox-casd')]
- casd_args.append('--bind=unix:' + self._casd_socket_path)
+ casd_args.append('--bind=' + self._casd_socket_connectstr)
if cache_quota is not None:
casd_args.append('--quota-high={}'.format(int(cache_quota)))
@@ -82,7 +83,12 @@ class CASCache():
if protect_session_blobs:
casd_args.append('--protect-session-blobs')
- casd_args.append(path)
+ # casd_args.append('--verbose')
+
+ # print("casd path:", os.path.abspath(path))
+ casd_args.append( os.path.abspath(path))
+ # casd_logfile = open("casd.logfile", "w")
+ # self._casd_process = subprocess.Popen(casd_args, cwd=path, stdout=casd_logfile, stderr=casd_logfile)
self._casd_process = subprocess.Popen(casd_args, cwd=path)
self._casd_start_time = time.time()
else:
@@ -107,7 +113,7 @@ class CASCache():
assert self._casd_process, "CASCache was instantiated without buildbox-casd"
if not self._casd_channel:
- self._casd_channel = grpc.insecure_channel('unix:' + self._casd_socket_path)
+ self._casd_channel = grpc.insecure_channel(self._casd_socket_connectstr)
self._casd_cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self._casd_channel)
self._local_cas = local_cas_pb2_grpc.LocalContentAddressableStorageStub(self._casd_channel)
@@ -200,7 +206,8 @@ class CASCache():
self._casd_process.wait(timeout=15)
self._casd_process = None
- shutil.rmtree(self._casd_socket_tempdir)
+ # TODO: only rmtree if not on Windows
+ # shutil.rmtree(self._casd_socket_tempdir)
# contains():
#
@@ -248,7 +255,8 @@ class CASCache():
with open(path, 'rb') as f:
directory.ParseFromString(f.read())
if update_mtime:
- os.utime(f.fileno())
+ # Seems like this is not supported on Windows.
+ os.utime(f.name)
# Optionally check presence of files
if with_files:
@@ -394,10 +402,14 @@ class CASCache():
with contextlib.ExitStack() as stack:
if path is None:
- tmp = stack.enter_context(self._temporary_object())
- tmp.write(buffer)
- tmp.flush()
- path = tmp.name
+ tmpname = stack.enter_context(self._temporary_object())
+ with open(tmpname, "wb") as tmp:
+ tmp.write(buffer)
+ tmp.flush() # Not necessary?
+ path = tmpname
+ # print("Temporary object:", path)
+
+ path = os.path.abspath(path)
request = local_cas_pb2.CaptureFilesRequest()
if instance_name:
@@ -419,6 +431,8 @@ class CASCache():
raise CASCacheError("Failed to capture blob {}: {}".format(path, blob_response.status.code))
digest.CopyFrom(blob_response.digest)
+ # print("Saved temporary object:", path)
+
return digest
# set_ref():
@@ -752,10 +766,10 @@ class CASCache():
# Create a named temporary file with 0o0644 access rights.
@contextlib.contextmanager
def _temporary_object(self):
- with utils._tempnamedfile(dir=self.tmpdir) as f:
- os.chmod(f.name,
+ with utils._tempnamedfile_name(dir=self.tmpdir) as fname:
+ os.chmod(fname,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
- yield f
+ yield fname
# _ensure_blob():
#
[buildstream] 16/25: WIP: job.py: no add_reader for files on win32
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b05dfd43ab9fe105a042c9f49df9462334502cc5
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Aug 29 12:55:29 2019 +0100
WIP: job.py: no add_reader for files on win32
---
src/buildstream/_scheduler/jobs/job.py | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 513af70..c769d0f 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -608,19 +608,21 @@ class Job():
#
# http://bugs.python.org/issue3831
#
- if not self._listening:
- self._scheduler.loop.add_reader(
- self._queue._reader.fileno(), self._parent_recv)
- self._listening = True
+ # if not self._listening:
+ # self._scheduler.loop.add_reader(
+ # self._queue._reader.fileno(), self._parent_recv)
+ # self._listening = True
+ pass
# _parent_stop_listening()
#
# Stops listening on the message queue
#
def _parent_stop_listening(self):
- if self._listening:
- self._scheduler.loop.remove_reader(self._queue._reader.fileno())
- self._listening = False
+ # if self._listening:
+ # self._scheduler.loop.remove_reader(self._queue._reader.fileno())
+ # self._listening = False
+ pass
# ChildJob()
[buildstream] 06/25: _signals.terminator: "outermost" ->
"is_outermost"
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f955634ff62a04c1cecaf14cfdc7b85ec6ad8186
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 8 09:59:18 2019 +0100
_signals.terminator: "outermost" -> "is_outermost"
---
src/buildstream/_signals.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/buildstream/_signals.py b/src/buildstream/_signals.py
index ec598fc..a7f32d6 100644
--- a/src/buildstream/_signals.py
+++ b/src/buildstream/_signals.py
@@ -86,16 +86,16 @@ def terminator(terminate_func):
yield
return
- outermost = bool(not terminator_stack)
+ is_outermost = bool(not terminator_stack)
terminator_stack.append(terminate_func)
- if outermost:
+ if is_outermost:
original_handler = signal.signal(signal.SIGTERM, terminator_handler)
try:
yield
finally:
- if outermost:
+ if is_outermost:
signal.signal(signal.SIGTERM, original_handler)
terminator_stack.pop()
[buildstream] 05/25: _signals.suspendable:"outermost" ->
"is_outermost"
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3c67037b0952218da5eaf98b145ebd1d3145ea25
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Tue Oct 8 09:24:25 2019 +0100
_signals.suspendable:"outermost" -> "is_outermost"
---
src/buildstream/_signals.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/buildstream/_signals.py b/src/buildstream/_signals.py
index 31982c1..ec598fc 100644
--- a/src/buildstream/_signals.py
+++ b/src/buildstream/_signals.py
@@ -146,17 +146,17 @@ def suspend_handler(sig, frame):
def suspendable(suspend_callback, resume_callback):
global suspendable_stack # pylint: disable=global-statement
- outermost = bool(not suspendable_stack)
+ is_outermost = bool(not suspendable_stack)
suspender = Suspender(suspend_callback, resume_callback)
suspendable_stack.append(suspender)
- if outermost:
+ if is_outermost:
original_stop = signal.signal(signal.SIGTSTP, suspend_handler)
try:
yield
finally:
- if outermost:
+ if is_outermost:
signal.signal(signal.SIGTSTP, original_stop)
suspendable_stack.pop()
[buildstream] 24/25: TEMP: ignore local windows files
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c7c324f91fc31a4d785f1702f155b94b33c74b3b
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Aug 29 10:59:03 2019 +0100
TEMP: ignore local windows files
---
.gitignore | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/.gitignore b/.gitignore
index 2a00671..9940b62 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,7 @@
+*.pyd
+*.swp
+*.orig
+
# Compiled python modules.
__pycache__
[buildstream] 15/25: WIP: win32: WINDOWS.md: repro instructions
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch aevri/win32_minimal
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 22bb841e9a35adefad6924d2ee7e4f0c240e823d
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Apr 18 11:16:51 2019 +0100
WIP: win32: WINDOWS.md: repro instructions
---
WINDOWS.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 59 insertions(+)
diff --git a/WINDOWS.md b/WINDOWS.md
new file mode 100644
index 0000000..46e3655
--- /dev/null
+++ b/WINDOWS.md
@@ -0,0 +1,59 @@
+Running on Windows
+==================
+
+This is a temporary doc tied to the lifetime of the `aevri/win32` branch,
+intended to help you repro the results.
+
+Installation
+------------
+
+First, make sure you have Python 3 installed.
+
+Next, you probably want to create a venv to install BuildStream into, for
+experimentation.
+
+Then, clone and install BuildStream:
+
+ git clone --branch aevri/win32 https://gitlab.com/buildstream/buildstream.git
+ pip install -e ./buildstream
+
+Next, install some additional dependencies for proper display:
+
+ pip install colorama windows-curses
+
+Finally, make sure you have the build tools installed:
+
+- Download the installer from: https://visualstudio.microsoft.com/downloads/#build-tools-for-visual-studio-2017
+- Run the installer.
+- Select to install "Visual C++ build tools". Possibly need to include these
+ optional items:
+ - Windows 10 SDK
+ - Visual C++ tools for CMake
+ - Testing tools core feature - Build Tools
+
+Hello World
+-----------
+
+Here is how to build the "Hello World" example.
+
+First, launch a "Developer Command Prompt for VS 2017". This ensures that you
+have the correct environment variables for building. The next instructions
+assume you are running inside this prompt.
+
+Next, make sure you have activated any virtual environment for BuildStream.
+
+Then, change directory to the buildstream git repository.
+
+Finally, build and run like so:
+
+ bst --help
+
+ cd doc\examples\running-commands
+
+ bst show hello.bst
+
+ bst build hello.bst
+
+ bst artifact checkout hello.bst --directory checkout
+ cd checkout
+ hello.exe