[buildstream] 02/04: tests/.../ftp_server: be spawn friendly

[tv...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch aevri/enable_spawn_ci_6
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 6fdfed345d7b92ac90c7e34b55f16b3dde0d1d11
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Fri Oct 25 15:20:26 2019 +0100

    tests/.../ftp_server: be spawn friendly
    
    Make SimpleFtpServer work when using the 'spawn' method of
    multiprocessing.
    
    Instead of trying to pickle the FTPServer and friends to the new
    process, buffer any config calls and then start the server only in the
    new process.
---
 tests/testutils/ftp_server.py | 74 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 59 insertions(+), 15 deletions(-)

diff --git a/tests/testutils/ftp_server.py b/tests/testutils/ftp_server.py
index 52c05f8..dbe3468 100644
--- a/tests/testutils/ftp_server.py
+++ b/tests/testutils/ftp_server.py
@@ -5,28 +5,72 @@ from pyftpdlib.handlers import FTPHandler
 from pyftpdlib.servers import FTPServer
 
 
-class SimpleFtpServer(multiprocessing.Process):
+class SimpleFtpServer():
+    # pylint: disable=attribute-defined-outside-init
+
     def __init__(self):
-        super().__init__()
-        self.authorizer = DummyAuthorizer()
-        handler = FTPHandler
-        handler.authorizer = self.authorizer
-        self.server = FTPServer(('127.0.0.1', 0), handler)
+        self._reset()
+
+    def _reset(self):
+        self._process = None
+        self._port = None
+        self._anonymous_dirs = []
+        self._user_list = []
+
+    def start(self):
+        assert self._process is None, "Server already running."
+        queue = multiprocessing.SimpleQueue()
 
-    def run(self):
-        self.server.serve_forever()
+        self._process = multiprocessing.Process(
+            target=_run_server,
+            args=(queue, self._anonymous_dirs, self._user_list),
+        )
+
+        self._process.start()
+        self._port = queue.get()
 
     def stop(self):
-        self.server.close_all()
-        self.server.close()
-        self.terminate()
-        self.join()
+        assert self._process is not None, "Server not running."
+
+        # Note that when spawning, terminating in this way will cause us to
+        # leak semaphores. This will lead to a warning from Python's semaphore
+        # tracker, which will clean them up for us.
+        #
+        # We could prevent this warning by using alternative methods that would
+        # let the _run_server() function finish, the extra complication doesn't
+        # seem worth it for this test class.
+        #
+        self._process.terminate()
+
+        self._process.join()
+        self._reset()
 
     def allow_anonymous(self, cwd):
-        self.authorizer.add_anonymous(cwd)
+        assert self._process is None, "Can't modify server after start()."
+        self._anonymous_dirs.append(cwd)
 
     def add_user(self, user, password, cwd):
-        self.authorizer.add_user(user, password, cwd, perm='elradfmwMT')
+        assert self._process is None, "Can't modify server after start()."
+        self._user_list.append((user, password, cwd))
 
     def base_url(self):
-        return 'ftp://127.0.0.1:{}'.format(self.server.address[1])
+        assert self._port is not None
+        return 'ftp://127.0.0.1:{}'.format(self._port)
+
+
+def _run_server(queue, anonymous_dirs, user_list):
+    authorizer = DummyAuthorizer()
+    handler = FTPHandler
+    handler.authorizer = authorizer
+
+    for cwd in anonymous_dirs:
+        authorizer.add_anonymous(cwd)
+
+    for user, password, cwd in user_list:
+        authorizer.add_user(user, password, cwd, perm='elradfmwMT')
+
+    server = FTPServer(('127.0.0.1', 0), handler)
+
+    port = server.address[1]
+    queue.put(port)
+    server.serve_forever(handle_exit=True)