You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2019/01/16 20:11:30 UTC
[airavata-django-portal] branch master updated: AIRAVATA-2692
Switch to thrift client connection pool
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
The following commit(s) were added to refs/heads/master by this push:
new dabc08f AIRAVATA-2692 Switch to thrift client connection pool
dabc08f is described below
commit dabc08f52b938fb5c8f6e29d849fa3bc4a6811f7
Author: Marcus Christie <ma...@iu.edu>
AuthorDate: Wed Jan 16 15:06:06 2019 -0500
AIRAVATA-2692 Switch to thrift client connection pool
---
django_airavata/apps/auth/iam_admin_client.py | 44 +++++------
django_airavata/apps/workspace/signals.py | 7 +-
django_airavata/middleware.py | 32 +++-----
django_airavata/settings.py | 5 ++
django_airavata/utils.py | 110 ++++++++++++++++++++++++++
requirements.txt | 1 +
setup.py | 1 +
7 files changed, 152 insertions(+), 48 deletions(-)
diff --git a/django_airavata/apps/auth/iam_admin_client.py b/django_airavata/apps/auth/iam_admin_client.py
index 6877d8a..7783ce7 100644
--- a/django_airavata/apps/auth/iam_admin_client.py
+++ b/django_airavata/apps/auth/iam_admin_client.py
@@ -4,7 +4,7 @@ Wrapper around the IAM Admin Services client.
import logging
-from django_airavata.utils import get_iam_admin_client
+from django_airavata.utils import iamadmin_client_pool
from . import utils
@@ -12,42 +12,36 @@ logger = logging.getLogger(__name__)
def is_username_available(username):
- with get_iam_admin_client() as iam_admin_client:
- authz_token = utils.get_service_account_authz_token()
- return iam_admin_client.isUsernameAvailable(authz_token, username)
+ authz_token = utils.get_service_account_authz_token()
+ return iamadmin_client_pool.isUsernameAvailable(authz_token, username)
def register_user(username, email_address, first_name, last_name, password):
- with get_iam_admin_client() as iam_admin_client:
- authz_token = utils.get_service_account_authz_token()
- return iam_admin_client.registerUser(
- authz_token,
- username,
- email_address,
- first_name,
- last_name,
- password)
+ authz_token = utils.get_service_account_authz_token()
+ return iamadmin_client_pool.registerUser(
+ authz_token,
+ username,
+ email_address,
+ first_name,
+ last_name,
+ password)
def is_user_enabled(username):
- with get_iam_admin_client() as iam_admin_client:
- authz_token = utils.get_service_account_authz_token()
- return iam_admin_client.isUserEnabled(authz_token, username)
+ authz_token = utils.get_service_account_authz_token()
+ return iamadmin_client_pool.isUserEnabled(authz_token, username)
def enable_user(username):
- with get_iam_admin_client() as iam_admin_client:
- authz_token = utils.get_service_account_authz_token()
- return iam_admin_client.enableUser(authz_token, username)
+ authz_token = utils.get_service_account_authz_token()
+ return iamadmin_client_pool.enableUser(authz_token, username)
def is_user_exist(username):
- with get_iam_admin_client() as iam_admin_client:
- authz_token = utils.get_service_account_authz_token()
- return iam_admin_client.isUserExist(authz_token, username)
+ authz_token = utils.get_service_account_authz_token()
+ return iamadmin_client_pool.isUserExist(authz_token, username)
def get_user(username):
- with get_iam_admin_client() as iam_admin_client:
- authz_token = utils.get_service_account_authz_token()
- return iam_admin_client.getUser(authz_token, username)
+ authz_token = utils.get_service_account_authz_token()
+ return iamadmin_client_pool.getUser(authz_token, username)
diff --git a/django_airavata/apps/workspace/signals.py b/django_airavata/apps/workspace/signals.py
index 4eac959..3886011 100644
--- a/django_airavata/apps/workspace/signals.py
+++ b/django_airavata/apps/workspace/signals.py
@@ -6,7 +6,7 @@ from django.contrib.auth.signals import user_logged_in
from django.dispatch import receiver
from django_airavata.apps.auth.utils import get_authz_token
-from django_airavata.utils import get_user_profile_client
+from django_airavata.utils import user_profile_client_pool
log = logging.getLogger(__name__)
@@ -15,6 +15,5 @@ log = logging.getLogger(__name__)
def initialize_user_profile(sender, request, user, **kwargs):
"""Initialize user profile in Airavata in case this is a new user."""
authz_token = get_authz_token(request)
- with get_user_profile_client() as user_profile_client:
- user_profile_client.initializeUserProfile(authz_token)
- log.debug("initialized user profile for {}".format(user.username))
+ user_profile_client_pool.initializeUserProfile(authz_token)
+ log.debug("initialized user profile for {}".format(user.username))
diff --git a/django_airavata/middleware.py b/django_airavata/middleware.py
index f7e31a6..e3bb19a 100644
--- a/django_airavata/middleware.py
+++ b/django_airavata/middleware.py
@@ -14,14 +14,13 @@ def airavata_client(get_response):
# If user is logged in create an airavata api client for the request
if request.user.is_authenticated:
try:
- with utils.get_airavata_client() as airavata_client:
- request.airavata_client = airavata_client
- response = get_response(request)
+ request.airavata_client = utils.airavata_api_client_pool
+ response = get_response(request)
except utils.ThriftConnectionException as e:
logger.exception(
"Failed to open thrift connection to API server")
- # if request.airavata_client is None, this will indicate to view
- # code that the API server is down
+ # if request.airavata_client is None, this will indicate to
+ # view code that the API server is down
request.airavata_client = None
response = get_response(request)
else:
@@ -40,9 +39,8 @@ def sharing_client(get_response):
# If user is logged in create a sharing registry client for the request
if request.user.is_authenticated:
try:
- with utils.get_sharing_client() as sharing_client:
- request.sharing_client = sharing_client
- response = get_response(request)
+ request.sharing_client = utils.sharing_api_client_pool
+ response = get_response(request)
except utils.ThriftConnectionException as e:
logger.exception("Failed to open thrift connection to"
"Sharing Registry server")
@@ -70,17 +68,13 @@ def profile_service_client(get_response):
# If user is logged in create an profile service client for the request
if request.user.is_authenticated:
try:
- with utils.get_group_manager_client() as group_manager_client, \
- utils.get_iam_admin_client() as iam_admin_client, \
- utils.get_tenant_profile_client() as tenant_profile_client, \
- utils.get_user_profile_client() as user_profile_client:
- request.profile_service = {
- 'group_manager': group_manager_client,
- 'iam_admin': iam_admin_client,
- 'tenant_profile': tenant_profile_client,
- 'user_profile': user_profile_client,
- }
- response = get_response(request)
+ request.profile_service = {
+ 'group_manager': utils.group_manager_client_pool,
+ 'iam_admin': utils.iamadmin_client_pool,
+ 'tenant_profile': utils.tenant_profile_client_pool,
+ 'user_profile': utils.user_profile_client_pool,
+ }
+ response = get_response(request)
except utils.ThriftConnectionException as e:
logger.exception("Failed to open thrift connection to "
"Profile Service server")
diff --git a/django_airavata/settings.py b/django_airavata/settings.py
index 4128c1a..987b3c0 100644
--- a/django_airavata/settings.py
+++ b/django_airavata/settings.py
@@ -211,6 +211,11 @@ AUTHENTICATION_OPTIONS = {
# ]
}
+# Seconds each connection in the pool is able to stay alive. If open connection
+# has lived longer than this period, it will be closed.
+# (https://github.com/Thriftpy/thrift_connector)
+THRIFT_CLIENT_POOL_KEEPALIVE = 10
+
LOGGING = {
'version': 1,
'handlers': {
diff --git a/django_airavata/utils.py b/django_airavata/utils.py
index e730bd2..c44d2a2 100644
--- a/django_airavata/utils.py
+++ b/django_airavata/utils.py
@@ -1,6 +1,7 @@
import logging
from contextlib import contextmanager
+import thrift_connector.connection_pool as connection_pool
from django.conf import settings
from thrift.protocol import TBinaryProtocol
from thrift.protocol.TMultiplexedProtocol import TMultiplexedProtocol
@@ -184,3 +185,112 @@ def get_thrift_client(host, port, is_secure, client_generator):
host, port, is_secure)
log.debug(msg)
raise ThriftConnectionException(msg) from e
+
+
+class CustomThriftClient(connection_pool.ThriftClient):
+ secure = False
+ validate = False
+
+ @classmethod
+ def get_socket_factory(cls):
+ if not cls.secure:
+ return super().get_socket_factory()
+ else:
+ def factory(host, port):
+ return TSSLSocket.TSSLSocket(host, port, validate=cls.validate)
+ return factory
+
+ def ping(self):
+ try:
+ self.client.getAPIVersion()
+ except Exception:
+ log.exception("getAPIVersion failed")
+ raise
+
+
+class MultiplexThriftClientMixin:
+ service_name = None
+
+ @classmethod
+ def get_protoco_factory(cls):
+ def factory(transport):
+ protocol = TBinaryProtocol.TBinaryProtocol(transport)
+ multiplex_prot = TMultiplexedProtocol(protocol, cls.service_name)
+ return multiplex_prot
+ return factory
+
+
+class AiravataAPIThriftClient(CustomThriftClient):
+ secure = settings.AIRAVATA_API_SECURE
+
+
+class GroupManagerServiceThriftClient(MultiplexThriftClientMixin,
+ CustomThriftClient):
+ service_name = GROUP_MANAGER_CPI_NAME
+ secure = settings.PROFILE_SERVICE_SECURE
+
+
+class IAMAdminServiceThriftClient(MultiplexThriftClientMixin,
+ CustomThriftClient):
+ service_name = IAM_ADMIN_SERVICES_CPI_NAME
+ secure = settings.PROFILE_SERVICE_SECURE
+
+
+class TenantProfileServiceThriftClient(MultiplexThriftClientMixin,
+ CustomThriftClient):
+ service_name = TENANT_PROFILE_CPI_NAME
+ secure = settings.PROFILE_SERVICE_SECURE
+
+
+class UserProfileServiceThriftClient(MultiplexThriftClientMixin,
+ CustomThriftClient):
+ service_name = USER_PROFILE_CPI_NAME
+ secure = settings.PROFILE_SERVICE_SECURE
+
+
+class SharingAPIThriftClient(CustomThriftClient):
+ secure = settings.SHARING_API_SECURE
+
+
+airavata_api_client_pool = connection_pool.ClientPool(
+ Airavata,
+ settings.AIRAVATA_API_HOST,
+ settings.AIRAVATA_API_PORT,
+ connection_class=AiravataAPIThriftClient,
+ keepalive=settings.THRIFT_CLIENT_POOL_KEEPALIVE
+)
+group_manager_client_pool = connection_pool.ClientPool(
+ GroupManagerService,
+ settings.PROFILE_SERVICE_HOST,
+ settings.PROFILE_SERVICE_PORT,
+ connection_class=GroupManagerServiceThriftClient,
+ keepalive=settings.THRIFT_CLIENT_POOL_KEEPALIVE
+)
+iamadmin_client_pool = connection_pool.ClientPool(
+ IamAdminServices,
+ settings.PROFILE_SERVICE_HOST,
+ settings.PROFILE_SERVICE_PORT,
+ connection_class=IAMAdminServiceThriftClient,
+ keepalive=settings.THRIFT_CLIENT_POOL_KEEPALIVE
+)
+tenant_profile_client_pool = connection_pool.ClientPool(
+ TenantProfileService,
+ settings.PROFILE_SERVICE_HOST,
+ settings.PROFILE_SERVICE_PORT,
+ connection_class=TenantProfileServiceThriftClient,
+ keepalive=settings.THRIFT_CLIENT_POOL_KEEPALIVE
+)
+user_profile_client_pool = connection_pool.ClientPool(
+ UserProfileService,
+ settings.PROFILE_SERVICE_HOST,
+ settings.PROFILE_SERVICE_PORT,
+ connection_class=UserProfileServiceThriftClient,
+ keepalive=settings.THRIFT_CLIENT_POOL_KEEPALIVE
+)
+sharing_api_client_pool = connection_pool.ClientPool(
+ SharingRegistryService,
+ settings.SHARING_API_HOST,
+ settings.SHARING_API_PORT,
+ connection_class=SharingAPIThriftClient,
+ keepalive=settings.THRIFT_CLIENT_POOL_KEEPALIVE
+)
diff --git a/requirements.txt b/requirements.txt
index 2c3c63c..bdbdda9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@ Django==1.11.16
requests==2.13.0
requests-oauthlib==0.7.0
thrift==0.10.0
+thrift_connector==0.24
djangorestframework==3.8.2
wagtail==2.0
diff --git a/setup.py b/setup.py
index 68a4c63..1760221 100644
--- a/setup.py
+++ b/setup.py
@@ -26,6 +26,7 @@ setup(
'requests',
'requests-oauthlib',
'thrift',
+ 'thrift_connector',
'wagtail',
'wagtailfontawesome'
],