You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2018/12/18 04:20:21 UTC
[GitHub] stale[bot] closed pull request #2131: [AIRFLOW-946] call commands
with virtualenv if available
stale[bot] closed pull request #2131: [AIRFLOW-946] call commands with virtualenv if available
URL: https://github.com/apache/incubator-airflow/pull/2131
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index eb96e77537..0119a3e861 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -56,6 +56,7 @@
from airflow.ti_deps.dep_context import (DepContext, SCHEDULER_DEPS)
from airflow.utils import cli as cli_utils
from airflow.utils import db as db_utils
+from airflow.utils.file import use_virtualenv
from airflow.utils.net import get_hostname
from airflow.utils.log.logging_mixin import (LoggingMixin, redirect_stderr,
redirect_stdout)
@@ -776,7 +777,7 @@ def webserver(args):
'''.format(**locals())))
run_args = [
- 'gunicorn',
+ use_virtualenv('gunicorn'),
'-w', str(num_workers),
'-k', str(args.workerclass),
'-t', str(worker_timeout),
@@ -802,6 +803,7 @@ def webserver(args):
run_args += ["airflow." + webserver_module + ".app:cached_app()"]
gunicorn_master_proc = None
+ env = os.environ.copy()
def kill_proc(dummy_signum, dummy_frame):
gunicorn_master_proc.terminate()
@@ -830,7 +832,7 @@ def monitor_gunicorn(gunicorn_master_proc):
},
)
with ctx:
- subprocess.Popen(run_args, close_fds=True)
+ subprocess.Popen(run_args, env=env, close_fds=True)
# Reading pid file directly, since Popen#pid doesn't
# seem to return the right value with DaemonContext.
@@ -849,7 +851,7 @@ def monitor_gunicorn(gunicorn_master_proc):
stdout.close()
stderr.close()
else:
- gunicorn_master_proc = subprocess.Popen(run_args, close_fds=True)
+ gunicorn_master_proc = subprocess.Popen(run_args, env=env, close_fds=True)
signal.signal(signal.SIGINT, kill_proc)
signal.signal(signal.SIGTERM, kill_proc)
@@ -943,7 +945,8 @@ def worker(args):
stderr=stderr,
)
with ctx:
- sp = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True)
+ sp = subprocess.Popen(use_virtualenv(['airflow', 'serve_logs']), env=env,
+ close_fds=True)
worker.run(**options)
sp.kill()
@@ -953,7 +956,8 @@ def worker(args):
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
- sp = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True)
+ sp = subprocess.Popen(use_virtualenv(['airflow', 'serve_logs']), env=env,
+ close_fds=True)
worker.run(**options)
sp.kill()
@@ -1143,7 +1147,8 @@ def flower(args):
flower_conf = '--conf=' + args.flower_conf
if args.daemon:
- pid, stdout, stderr, log_file = setup_locations("flower", args.pid, args.stdout, args.stderr, args.log_file)
+ pid, stdout, stderr, log_file = setup_locations(
+ "flower", args.pid, args.stdout, args.stderr, args.log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
@@ -1154,8 +1159,8 @@ def flower(args):
)
with ctx:
- os.execvp("flower", ['flower', '-b',
- broka, address, port, api, flower_conf, url_prefix])
+ os.execvp(use_virtualenv('flower'),
+ ['flower', '-b', broka, address, port, api, flower_conf, url_prefix])
stdout.close()
stderr.close()
@@ -1163,8 +1168,8 @@ def flower(args):
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
- os.execvp("flower", ['flower', '-b',
- broka, address, port, api, flower_conf, url_prefix])
+ os.execvp(use_virtualenv('flower'),
+ ['flower', '-b', broka, address, port, api, flower_conf, url_prefix])
@cli_utils.action_logging
diff --git a/airflow/models.py b/airflow/models.py
index afcacd126b..ac4814c333 100755
--- a/airflow/models.py
+++ b/airflow/models.py
@@ -78,6 +78,7 @@
from airflow.utils.db import provide_session
from airflow.utils.decorators import apply_defaults
from airflow.utils.email import send_email
+from airflow.utils.file import use_virtualenv
from airflow.utils.helpers import (
as_tuple, is_container, is_in, validate_key, pprinttable)
from airflow.utils.operator_resources import Resources
@@ -1016,7 +1017,8 @@ def generate_command(dag_id,
:return: shell command that can be used to run the task instance
"""
iso = execution_date.isoformat()
- cmd = ["airflow", "run", str(dag_id), str(task_id), str(iso)]
+ cmd = [use_virtualenv("airflow"), "run", str(dag_id), str(task_id),
+ str(iso)]
cmd.extend(["--mark_success"]) if mark_success else None
cmd.extend(["--pickle", str(pickle_id)]) if pickle_id else None
cmd.extend(["--job_id", str(job_id)]) if job_id else None
diff --git a/airflow/utils/file.py b/airflow/utils/file.py
index 352755e405..49791786ad 100644
--- a/airflow/utils/file.py
+++ b/airflow/utils/file.py
@@ -18,6 +18,7 @@
import errno
import os
import shutil
+import sys
from tempfile import mkdtemp
from contextlib import contextmanager
@@ -41,7 +42,6 @@ def mkdirs(path, mode):
"""
Creates the directory specified by path, creating intermediate directories
as necessary. If directory already exists, this is a no-op.
-
:param path: The directory to create
:type path: str
:param mode: The mode to give to the directory e.g. 0o755, ignores umask
@@ -55,3 +55,16 @@ def mkdirs(path, mode):
raise
finally:
os.umask(o_umask)
+
+
+def use_virtualenv(command):
+ """
+ If we're in a virtualenv, ensure we call the given command using the
+ its virtualenv wrapped script. Otherwise, just return command.
+
+ Example: gunicorn -> /path/to/venv/bin/gunicorn
+ """
+ if hasattr(sys, 'real_prefix'):
+ return os.path.join(os.path.dirname(sys.executable), command)
+
+ return command
diff --git a/tests/utils/file.py b/tests/utils/file.py
new file mode 100644
index 0000000000..3c99dd86fb
--- /dev/null
+++ b/tests/utils/file.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from mock import patch
+
+from airflow.utils.file import use_virtualenv
+
+
+class VirtualEnvTest(unittest.TestCase):
+
+ @patch('airflow.utils.file.sys', executable='/usr/bin/python')
+ def test_not_using_virtualenv(self, mock_sys):
+ if hasattr(mock_sys, 'real_prefix'):
+ del mock_sys.real_prefix
+ self.assertEqual('gunicorn', use_virtualenv('gunicorn'))
+
+ @patch('airflow.utils.file.sys', real_prefix='/usr',
+ executable='/path/to/venv/bin/python')
+ def test_using_virtualenv(self, mock_sys):
+ self.assertEqual('/path/to/venv/bin/gunicorn',
+ use_virtualenv('gunicorn'))
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services