You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by ra...@apache.org on 2017/05/22 12:17:30 UTC
[03/19] incubator-ariatosca git commit: ARIA-215 Refactor
plugin-related code into PluginManager
ARIA-215 Refactor plugin-related code into PluginManager
Refactored plugin-related code from ProcessExecutor into PluginManager.
Additionally, renamed plugin_prefix to plugin_dir in PluginManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/3e1ed14c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/3e1ed14c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/3e1ed14c
Branch: refs/heads/ARIA-208-Missing-back-refrences-for-models
Commit: 3e1ed14c00ea2c83fafdca8ec0e37817bea1d5e8
Parents: 45c158e
Author: Ran Ziv <ra...@gigaspaces.com>
Authored: Sun May 7 16:36:39 2017 +0300
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Sun May 7 17:27:36 2017 +0300
----------------------------------------------------------------------
aria/orchestrator/plugin.py | 36 +++++++++++-
aria/orchestrator/workflows/executor/process.py | 60 ++++++--------------
aria/utils/process.py | 47 +++++++++++++++
tests/utils/test_plugin.py | 6 +-
4 files changed, 100 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/orchestrator/plugin.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/plugin.py b/aria/orchestrator/plugin.py
index f99666c..8fbcf5a 100644
--- a/aria/orchestrator/plugin.py
+++ b/aria/orchestrator/plugin.py
@@ -23,6 +23,9 @@ from datetime import datetime
import wagon
from . import exceptions
+from ..utils import process as process_utils
+
+_IS_WIN = os.name == 'nt'
class PluginManager(object):
@@ -62,11 +65,40 @@ class PluginManager(object):
raise exceptions.PluginAlreadyExistsError(
'Plugin {0}, version {1} already exists'.format(plugin.package_name,
plugin.package_version))
- self._install_wagon(source=source, prefix=self.get_plugin_prefix(plugin))
+ self._install_wagon(source=source, prefix=self.get_plugin_dir(plugin))
self._model.plugin.put(plugin)
return plugin
- def get_plugin_prefix(self, plugin):
+ def load_plugin(self, plugin, env=None):
+ """
+ Load the plugin into an environment.
+ Loading the plugin means the plugin's code and binaries paths will be appended to the
+ environment's PATH and PYTHONPATH, thereby allowing usage of the plugin.
+ :param plugin: The plugin to load
+ :param env: The environment to load the plugin into; If `None`, os.environ will be used.
+ """
+ env = env or os.environ
+ plugin_dir = self.get_plugin_dir(plugin)
+
+ # Update PATH environment variable to include plugin's bin dir
+ bin_dir = 'Scripts' if _IS_WIN else 'bin'
+ process_utils.append_to_path(os.path.join(plugin_dir, bin_dir), env=env)
+
+ # Update PYTHONPATH environment variable to include plugin's site-packages
+ # directories
+ if _IS_WIN:
+ pythonpath_dirs = [os.path.join(plugin_dir, 'Lib', 'site-packages')]
+ else:
+ # In some linux environments, there will be both a lib and a lib64 directory
+ # with the latter, containing compiled packages.
+ pythonpath_dirs = [os.path.join(
+ plugin_dir, 'lib{0}'.format(b),
+ 'python{0}.{1}'.format(sys.version_info[0], sys.version_info[1]),
+ 'site-packages') for b in ('', '64')]
+
+ process_utils.append_to_pythonpath(*pythonpath_dirs, env=env)
+
+ def get_plugin_dir(self, plugin):
return os.path.join(
self._plugins_dir,
'{0}-{1}'.format(plugin.package_name, plugin.package_version))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index 8481406..f3daf04 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -47,13 +47,12 @@ from aria.storage import instrumentation
from aria.extension import process_executor
from aria.utils import (
imports,
- exceptions
+ exceptions,
+ process as process_utils
)
from aria.modeling import types as modeling_types
-_IS_WIN = os.name == 'nt'
-
_INT_FMT = 'I'
_INT_SIZE = struct.calcsize(_INT_FMT)
UPDATE_TRACKED_CHANGES_FAILED_STR = \
@@ -127,13 +126,7 @@ class ProcessExecutor(base.BaseExecutor):
with open(arguments_json_path, 'wb') as f:
f.write(pickle.dumps(self._create_arguments_dict(task)))
- env = os.environ.copy()
- # See _update_env for plugin_prefix usage
- if task.plugin_fk and self._plugin_manager:
- plugin_prefix = self._plugin_manager.get_plugin_prefix(task.plugin)
- else:
- plugin_prefix = None
- self._update_env(env=env, plugin_prefix=plugin_prefix)
+ env = self._construct_subprocess_env(task=task)
# Asynchronously start the operation in a subprocess
subprocess.Popen(
'{0} {1} {2}'.format(sys.executable, __file__, arguments_json_path),
@@ -156,40 +149,19 @@ class ProcessExecutor(base.BaseExecutor):
'context': task.context.serialization_dict,
}
- def _update_env(self, env, plugin_prefix):
- pythonpath_dirs = []
- # If this is a plugin operation, plugin prefix will point to where
- # This plugin is installed.
- # We update the environment variables that the subprocess will be started with based on it
- if plugin_prefix:
-
- # Update PATH environment variable to include plugin's bin dir
- bin_dir = 'Scripts' if _IS_WIN else 'bin'
- env['PATH'] = '{0}{1}{2}'.format(
- os.path.join(plugin_prefix, bin_dir),
- os.pathsep,
- env.get('PATH', ''))
-
- # Update PYTHONPATH environment variable to include plugin's site-packages
- # directories
- if _IS_WIN:
- pythonpath_dirs = [os.path.join(plugin_prefix, 'Lib', 'site-packages')]
- else:
- # In some linux environments, there will be both a lib and a lib64 directory
- # with the latter, containing compiled packages.
- pythonpath_dirs = [os.path.join(
- plugin_prefix, 'lib{0}'.format(b),
- 'python{0}.{1}'.format(sys.version_info[0], sys.version_info[1]),
- 'site-packages') for b in ('', '64')]
-
- # Add used supplied directories to injected PYTHONPATH
- pythonpath_dirs.extend(self._python_path)
-
- if pythonpath_dirs:
- env['PYTHONPATH'] = '{0}{1}{2}'.format(
- os.pathsep.join(pythonpath_dirs),
- os.pathsep,
- env.get('PYTHONPATH', ''))
+ def _construct_subprocess_env(self, task):
+ env = os.environ.copy()
+
+ if task.plugin_fk and self._plugin_manager:
+ # If this is a plugin operation,
+ # load the plugin on the subprocess env we're constructing
+ self._plugin_manager.load_plugin(task.plugin, env=env)
+
+ # Add user supplied directories to injected PYTHONPATH
+ if self._python_path:
+ process_utils.append_to_pythonpath(*self._python_path, env=env)
+
+ return env
def _listener(self):
# Notify __init__ method this thread has actually started
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/utils/process.py
----------------------------------------------------------------------
diff --git a/aria/utils/process.py b/aria/utils/process.py
new file mode 100644
index 0000000..9aeae67
--- /dev/null
+++ b/aria/utils/process.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+
+def append_to_path(*args, **kwargs):
+ """
+ Appends one or more paths to the system path of an environment.
+ The environment will be that of the current process unless another is passed using the
+ 'env' keyword argument.
+ :param args: paths to append
+ :param kwargs: 'env' may be used to pass a custom environment to use
+ """
+ _append_to_path('PATH', *args, **kwargs)
+
+
+def append_to_pythonpath(*args, **kwargs):
+ """
+ Appends one or more paths to the python path of an environment.
+ The environment will be that of the current process unless another is passed using the
+ 'env' keyword argument.
+ :param args: paths to append
+ :param kwargs: 'env' may be used to pass a custom environment to use
+ """
+ _append_to_path('PYTHONPATH', *args, **kwargs)
+
+
+def _append_to_path(path, *args, **kwargs):
+ env = kwargs.get('env') or os.environ
+ env[path] = '{0}{1}{2}'.format(
+ os.pathsep.join(args),
+ os.pathsep,
+ env.get(path, '')
+ )
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/tests/utils/test_plugin.py
----------------------------------------------------------------------
diff --git a/tests/utils/test_plugin.py b/tests/utils/test_plugin.py
index 3350247..c91d0c9 100644
--- a/tests/utils/test_plugin.py
+++ b/tests/utils/test_plugin.py
@@ -38,9 +38,9 @@ class TestPluginManager(object):
assert plugin.package_name == PACKAGE_NAME
assert plugin.package_version == PACKAGE_VERSION
assert plugin == model.plugin.get(plugin.id)
- plugin_prefix = os.path.join(plugins_dir, '{0}-{1}'.format(PACKAGE_NAME, PACKAGE_VERSION))
- assert os.path.isdir(plugin_prefix)
- assert plugin_prefix == plugin_manager.get_plugin_prefix(plugin)
+ plugin_dir = os.path.join(plugins_dir, '{0}-{1}'.format(PACKAGE_NAME, PACKAGE_VERSION))
+ assert os.path.isdir(plugin_dir)
+ assert plugin_dir == plugin_manager.get_plugin_dir(plugin)
def test_install_already_exits(self, plugin_manager, mock_plugin):
plugin_manager.install(mock_plugin)