You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/09/05 09:51:59 UTC
[incubator-dlab] branch DLAB-terraform updated: removed security
service configuration; refactored mongo parameters;
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-terraform
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-terraform by this push:
new 3390905 removed security service configuration; refactored mongo parameters;
3390905 is described below
commit 3390905003680aed71ed4c6be45df0ae05452f78
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Thu Sep 5 12:51:50 2019 +0300
removed security service configuration;
refactored mongo parameters;
---
.../src/general/lib/os/debian/ssn_lib.py | 57 +++----
.../src/general/lib/os/redhat/ssn_lib.py | 55 +++----
.../src/general/scripts/aws/ssn_configure.py | 182 ++++++++++++++++++---
.../src/general/scripts/azure/ssn_configure.py | 162 ++++++++++++++----
.../src/general/scripts/gcp/ssn_configure.py | 125 +++++++++++++-
.../src/ssn/scripts/configure_billing.py | 8 -
.../src/ssn/scripts/configure_mongo.py | 16 +-
.../src/ssn/scripts/configure_ui.py | 26 +--
.../src/ssn/templates/supervisor_svc.conf | 18 +-
services/self-service/self-service.yml | 42 +++--
10 files changed, 501 insertions(+), 190 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index 8784819..72fc5a9 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -27,6 +27,7 @@ import yaml
from dlab.fab import *
from dlab.meta_lib import *
import os
+import json
import traceback
import sys
@@ -170,7 +171,7 @@ def ensure_mongo():
def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
os_user, mongo_passwd, keystore_passwd, cloud_provider,
service_base_name, tag_resource_id, billing_tag, account_id, billing_bucket,
- aws_job_enabled, dlab_path, billing_enabled,
+ aws_job_enabled, dlab_path, billing_enabled, cloud_params,
authentication_file, offer_number, currency,
locale, region_info, ldap_login, tenant_id,
application_id, hostname, data_lake_name, subscription_id,
@@ -204,32 +205,34 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
try:
sudo('mkdir -p /var/log/application')
run('mkdir -p /tmp/yml_tmp/')
- for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
+ for service in ['self-service', 'provisioning-service', 'billing']:
jar = sudo('cd {0}{1}/lib/; find {1}*.jar -type f'.format(web_path, service))
sudo('ln -s {0}{2}/lib/{1} {0}{2}/{2}.jar '.format(web_path, jar, service))
sudo('cp {0}/webapp/{1}/conf/*.yml /tmp/yml_tmp/'.format(dlab_path, service))
+ # Replacing Keycloak and cloud parameters
+ for item in json.loads(cloud_params):
+ sudo('sed -i "s|{0}|{1}|g" /tmp/yml_tmp/self-service.yml'.format(
+ item['key'], item['value']))
+
if cloud_provider == 'azure':
- for config in ['self-service', 'security']:
- sudo('sed -i "s|<LOGIN_USE_LDAP>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config, ldap_login))
- sudo('sed -i "s|<LOGIN_TENANT_ID>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config, tenant_id))
- sudo('sed -i "s|<LOGIN_APPLICATION_ID>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config,
- application_id))
- sudo('sed -i "s|<DLAB_SUBSCRIPTION_ID>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config,
- subscription_id))
- sudo('sed -i "s|<MANAGEMENT_API_AUTH_FILE>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config, authentication_file))
- sudo('sed -i "s|<VALIDATE_PERMISSION_SCOPE>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(
- config, validate_permission_scope))
- sudo('sed -i "s|<LOGIN_APPLICATION_REDIRECT_URL>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config,
- hostname))
- sudo('sed -i "s|<LOGIN_PAGE>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config, hostname))
- if os.environ['azure_datalake_enable'] == 'true':
- permission_scope = 'subscriptions/{}/resourceGroups/{}/providers/Microsoft.DataLakeStore/accounts/{}/providers/Microsoft.Authorization/'.format(
- subscription_id, service_base_name, data_lake_name)
- else:
- permission_scope = 'subscriptions/{}/resourceGroups/{}/providers/Microsoft.Authorization/'.format(
- subscription_id, service_base_name
- )
- sudo('sed -i "s|<PERMISSION_SCOPE>|{}|g" /tmp/yml_tmp/security.yml'.format(permission_scope))
+ sudo('sed -i "s|<LOGIN_USE_LDAP>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(ldap_login))
+ sudo('sed -i "s|<LOGIN_TENANT_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(tenant_id))
+ sudo('sed -i "s|<LOGIN_APPLICATION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(application_id))
+ sudo('sed -i "s|<DLAB_SUBSCRIPTION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(subscription_id))
+ sudo('sed -i "s|<MANAGEMENT_API_AUTH_FILE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(
+ authentication_file))
+ sudo('sed -i "s|<VALIDATE_PERMISSION_SCOPE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(
+ validate_permission_scope))
+ sudo('sed -i "s|<LOGIN_APPLICATION_REDIRECT_URL>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(
+ hostname))
+ sudo('sed -i "s|<LOGIN_PAGE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(hostname))
+ # if os.environ['azure_datalake_enable'] == 'true':
+ # permission_scope = 'subscriptions/{}/resourceGroups/{}/providers/Microsoft.DataLakeStore/accounts/{}/providers/Microsoft.Authorization/'.format(
+ # subscription_id, service_base_name, data_lake_name)
+ # else:
+ # permission_scope = 'subscriptions/{}/resourceGroups/{}/providers/Microsoft.Authorization/'.format(
+ # subscription_id, service_base_name
+ # )
sudo('mv /tmp/yml_tmp/* ' + dlab_conf_dir)
sudo('rmdir /tmp/yml_tmp/')
except:
@@ -259,10 +262,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
'--usage {} ' \
'--cost {} ' \
'--resource_id {} ' \
- '--keycloak_realm_name {} ' \
- '--keycloak_auth_server_url {} ' \
- '--keycloak_client_name {} ' \
- '--keycloak_client_secret {} ' \
'--tags {}'.\
format(cloud_provider,
service_base_name,
@@ -286,10 +285,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
usage,
cost,
resource_id,
- os.environ['keycloak_realm_name'],
- os.environ['keycloak_auth_server_url'],
- os.environ['keycloak_client_name'],
- os.environ['keycloak_client_secret'],
tags)
sudo('python /tmp/configure_billing.py {}'.format(params))
try:
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index 2c4293f..b3022d2 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -27,6 +27,7 @@ import yaml
from dlab.fab import *
from dlab.meta_lib import *
import os
+import json
import sys
import traceback
@@ -196,7 +197,7 @@ def ensure_mongo():
def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
os_user, mongo_passwd, keystore_passwd, cloud_provider,
service_base_name, tag_resource_id, billing_tag, account_id, billing_bucket,
- aws_job_enabled, dlab_path, billing_enabled,
+ aws_job_enabled, dlab_path, billing_enabled, cloud_params,
authentication_file, offer_number, currency,
locale, region_info, ldap_login, tenant_id,
application_id, hostname, data_lake_name, subscription_id,
@@ -230,33 +231,31 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
try:
sudo('mkdir -p /var/log/application')
run('mkdir -p /tmp/yml_tmp/')
- for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
+ for service in ['self-service', 'provisioning-service', 'billing']:
jar = sudo('cd {0}{1}/lib/; find {1}*.jar -type f'.format(web_path, service))
sudo('ln -s {0}{2}/lib/{1} {0}{2}/{2}.jar '.format(web_path, jar, service))
sudo('cp {0}/webapp/{1}/conf/*.yml /tmp/yml_tmp/'.format(dlab_path, service))
+ # Replacing Keycloak and cloud parameters
+ for item in json.loads(cloud_params):
+ sudo('sed -i "s|{0}|{1}|g" /tmp/yml_tmp/self-service.yml'.format(
+ item['key'], item['value']))
if os.environ['conf_cloud_provider'] == 'azure':
- for config in ['self-service', 'security']:
- sudo('sed -i "s|<LOGIN_USE_LDAP>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config, ldap_login))
- sudo('sed -i "s|<LOGIN_TENANT_ID>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config, tenant_id))
- sudo('sed -i "s|<LOGIN_APPLICATION_ID>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config,
- application_id))
- sudo('sed -i "s|<DLAB_SUBSCRIPTION_ID>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config,
- subscription_id))
- sudo('sed -i "s|<MANAGEMENT_API_AUTH_FILE>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config, authentication_file))
- sudo('sed -i "s|<VALIDATE_PERMISSION_SCOPE>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(
- config, validate_permission_scope))
- sudo('sed -i "s|<LOGIN_APPLICATION_REDIRECT_URL>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config,
- hostname))
- sudo('sed -i "s|<LOGIN_PAGE>|{1}|g" /tmp/yml_tmp/{0}.yml'.format(config, hostname))
- if os.environ['azure_datalake_enable'] == 'true':
- permission_scope = 'subscriptions/{}/resourceGroups/{}/providers/Microsoft.DataLakeStore/accounts/{}/providers/Microsoft.Authorization/'.format(
- subscription_id, service_base_name, data_lake_name
- )
- else:
- permission_scope = 'subscriptions/{}/resourceGroups/{}/providers/Microsoft.Authorization/'.format(
- subscription_id, service_base_name
- )
- sudo('sed -i "s|<PERMISSION_SCOPE>|{}|g" /tmp/yml_tmp/security.yml'.format(permission_scope))
+ sudo('sed -i "s|<LOGIN_USE_LDAP>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(ldap_login))
+ sudo('sed -i "s|<LOGIN_TENANT_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(tenant_id))
+ sudo('sed -i "s|<LOGIN_APPLICATION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(application_id))
+ sudo('sed -i "s|<DLAB_SUBSCRIPTION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(subscription_id))
+ sudo('sed -i "s|<MANAGEMENT_API_AUTH_FILE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(authentication_file))
+ sudo('sed -i "s|<VALIDATE_PERMISSION_SCOPE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(validate_permission_scope))
+ sudo('sed -i "s|<LOGIN_APPLICATION_REDIRECT_URL>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(hostname))
+ sudo('sed -i "s|<LOGIN_PAGE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(hostname))
+ # if os.environ['azure_datalake_enable'] == 'true':
+ # permission_scope = 'subscriptions/{}/resourceGroups/{}/providers/Microsoft.DataLakeStore/accounts/{}/providers/Microsoft.Authorization/'.format(
+ # subscription_id, service_base_name, data_lake_name
+ # )
+ # else:
+ # permission_scope = 'subscriptions/{}/resourceGroups/{}/providers/Microsoft.Authorization/'.format(
+ # subscription_id, service_base_name
+ # )
sudo('mv /tmp/yml_tmp/* ' + os.environ['ssn_dlab_path'] + 'conf/')
sudo('rmdir /tmp/yml_tmp/')
except Exception as err:
@@ -289,10 +288,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
'--usage {} ' \
'--cost {} ' \
'--resource_id {} ' \
- '--keycloak_realm_name {} ' \
- '--keycloak_auth_server_url {} ' \
- '--keycloak_client_name {} ' \
- '--keycloak_client_secret {} ' \
'--tags {}'.\
format(cloud_provider,
service_base_name,
@@ -316,10 +311,6 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
usage,
cost,
resource_id,
- os.environ['keycloak_realm_name'],
- os.environ['keycloak_auth_server_url'],
- os.environ['keycloak_client_name'],
- os.environ['keycloak_client_secret'],
tags)
sudo('python /tmp/configure_billing.py {}'.format(params))
try:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
index 75daf41..6e2d2bb 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
@@ -313,30 +313,164 @@ if __name__ == "__main__":
sys.exit(1)
try:
- mongo_parameters = {
- "aws_region": os.environ['aws_region'],
- "aws_vpc_id": os.environ['aws_vpc_id'],
- "aws_subnet_id": os.environ['aws_subnet_id'],
- "conf_service_base_name": service_base_name,
- "aws_security_groups_ids": os.environ['aws_security_groups_ids'].replace(" ", ""),
- "conf_os_family": os.environ['conf_os_family'],
- "conf_tag_resource_id": os.environ['conf_tag_resource_id'],
- "conf_key_dir": os.environ['conf_key_dir'],
- "ssn_instance_size": os.environ['aws_ssn_instance_size'],
- "edge_instance_size": os.environ['aws_edge_instance_size']
- }
- if os.environ['conf_duo_vpc_enable'] == 'true':
- secondary_parameters = {
- "aws_notebook_vpc_id": os.environ['aws_vpc2_id'],
- "aws_notebook_subnet_id": os.environ['aws_subnet_id'],
- "aws_peering_id": os.environ['aws_peering_id']
+ # mongo_parameters = {
+ # "aws_region": os.environ['aws_region'],
+ # "aws_vpc_id": os.environ['aws_vpc_id'],
+ # "aws_subnet_id": os.environ['aws_subnet_id'],
+ # "conf_service_base_name": service_base_name,
+ # "aws_security_groups_ids": os.environ['aws_security_groups_ids'].replace(" ", ""),
+ # "conf_os_family": os.environ['conf_os_family'],
+ # "conf_tag_resource_id": os.environ['conf_tag_resource_id'],
+ # "conf_key_dir": os.environ['conf_key_dir'],
+ # "ssn_instance_size": os.environ['aws_ssn_instance_size'],
+ # "edge_instance_size": os.environ['aws_edge_instance_size']
+ # }
+ # if os.environ['conf_duo_vpc_enable'] == 'true':
+ # secondary_parameters = {
+ # "aws_notebook_vpc_id": os.environ['aws_vpc2_id'],
+ # "aws_notebook_subnet_id": os.environ['aws_subnet_id'],
+ # "aws_peering_id": os.environ['aws_peering_id']
+ # }
+ # else:
+ # secondary_parameters = {
+ # "aws_notebook_vpc_id": os.environ['aws_vpc_id'],
+ # "aws_notebook_subnet_id": os.environ['aws_subnet_id'],
+ # }
+ # mongo_parameters.update(secondary_parameters)
+ cloud_params = [
+ {
+ 'key': 'KEYCLOAK_REALM_NAME',
+ 'value': os.environ['keycloak_realm_name']
+ },
+ {
+ 'key': 'KEYCLOAK_AUTH_SERVER_URL',
+ 'value': os.environ['keycloak_auth_server_url']
+ },
+ {
+ 'key': 'KEYCLOAK_CLIENT_NAME',
+ 'value': os.environ['keycloak_client_name']
+ },
+ {
+ 'key': 'KEYCLOAK_CLIENT_SECRET',
+ 'value': os.environ['keycloak_client_secret']
+ },
+ {
+ 'key': 'CONF_OS',
+ 'value': os.environ['conf_os_family']
+ },
+ {
+ 'key': 'SERVICE_BASE_NAME',
+ 'value': os.environ['conf_service_base_name']
+ },
+ {
+ 'key': 'EDGE_INSTANCE_SIZE',
+ 'value': os.environ['aws_edge_instance_size']
+ },
+ {
+ 'key': 'SUBNET_ID',
+ 'value': os.environ['aws_subnet_id']
+ },
+ {
+ 'key': 'REGION',
+ 'value': os.environ['aws_region']
+ },
+ {
+ 'key': 'ZONE',
+ 'value': os.environ['aws_zone']
+ },
+ {
+ 'key': 'TAG_RESOURCE_ID',
+ 'value': os.environ['conf_tag_resource_id']
+ },
+ {
+ 'key': 'SG_IDS',
+ 'value': os.environ['aws_security_groups_ids']
+ },
+ {
+ 'key': 'SSN_INSTANCE_SIZE',
+ 'value': os.environ['aws_ssn_instance_size']
+ },
+ {
+ 'key': 'VPC_ID',
+ 'value': os.environ['aws_vpc_id']
+ },
+ {
+ 'key': 'CONF_KEY_DIR',
+ 'value': os.environ['conf_key_dir']
+ },
+ {
+ 'key': 'LDAP_HOST',
+ 'value': os.environ['ldap_host']
+ },
+ {
+ 'key': 'LDAP_DN',
+ 'value': os.environ['ldap_dn']
+ },
+ {
+ 'key': 'LDAP_OU',
+ 'value': os.environ['ldap_ou']
+ },
+ {
+ 'key': 'LDAP_USER',
+ 'value': os.environ['ldap_service_username']
+ },
+ {
+ 'key': 'LDAP_USER_PASSWORD',
+ 'value': os.environ['ldap_service_password']
+ },
+ {
+ 'key': 'AZURE_RESOURCE_GROUP_NAME',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_SSN_STORAGE_ACCOUNT_TAG',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_SHARED_STORAGE_ACCOUNT_TAG',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_DATALAKE_TAG',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_CLIENT_ID',
+ 'value': ''
}
+ ]
+ if os.environ['conf_duo_vpc_enable'] == 'true':
+ cloud_params.append(
+ {
+ 'key': 'SUBNET2_ID',
+ 'value': os.environ['aws_subnet_id']
+ })
+ cloud_params.append(
+ {
+ 'key': 'VPC2_ID',
+ 'value': os.environ['aws_vpc2_id']
+ })
+ cloud_params.append(
+ {
+ 'key': 'PEERING_ID',
+ 'value': os.environ['aws_peering_id']
+ })
else:
- secondary_parameters = {
- "aws_notebook_vpc_id": os.environ['aws_vpc_id'],
- "aws_notebook_subnet_id": os.environ['aws_subnet_id'],
- }
- mongo_parameters.update(secondary_parameters)
+ cloud_params.append(
+ {
+ 'key': 'SUBNET2_ID',
+ 'value': os.environ['aws_subnet_id']
+ })
+ cloud_params.append(
+ {
+ 'key': 'VPC2_ID',
+ 'value': os.environ['aws_vpc_id']
+ })
+ cloud_params.append(
+ {
+ 'key': 'PEERING_ID',
+ 'value': ''
+ })
logging.info('[CONFIGURE SSN INSTANCE UI]')
print('[CONFIGURE SSN INSTANCE UI]')
params = "--hostname {} " \
@@ -355,7 +489,7 @@ if __name__ == "__main__":
"--aws_job_enabled {} " \
"--report_path '{}' " \
"--billing_enabled {} " \
- "--mongo_parameters '{}' " \
+ "--cloud_params '{}' " \
"--dlab_id '{}' " \
"--usage_date {} " \
"--product {} " \
@@ -380,7 +514,7 @@ if __name__ == "__main__":
os.environ['aws_job_enabled'],
os.environ['aws_report_path'],
billing_enabled,
- json.dumps(mongo_parameters),
+ json.dumps(cloud_params),
os.environ['dlab_id'],
os.environ['usage_date'],
os.environ['product'],
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
index 78cd2a1..0a12d43 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
@@ -193,21 +193,125 @@ if __name__ == "__main__":
print('[CONFIGURE SSN INSTANCE UI]')
azure_auth_path = '/home/{}/keys/azure_auth.json'.format(ssn_conf['dlab_ssh_user'])
ldap_login = 'false'
- if os.environ['azure_datalake_enable'] == 'false':
- mongo_parameters = {
- "azure_resource_group_name": ssn_conf['resource_group_name'],
- "azure_region": ssn_conf['region'],
- "azure_vpc_name": ssn_conf['vpc_name'],
- "azure_subnet_name": ssn_conf['subnet_name'],
- "conf_service_base_name": ssn_conf['service_base_name'],
- "azure_security_group_name": ssn_conf['security_group_name'],
- "conf_os_family": os.environ['conf_os_family'],
- "conf_key_dir": os.environ['conf_key_dir'],
- "ssn_instance_size": os.environ['azure_ssn_instance_size'],
- "edge_instance_size": os.environ['azure_edge_instance_size'],
- "ssn_storage_account_tag_name": ssn_conf['ssn_storage_account_name'],
- "shared_storage_account_tag_name": ssn_conf['shared_storage_account_name']
+
+ cloud_params = [
+ {
+ 'key': 'KEYCLOAK_REALM_NAME',
+ 'value': os.environ['keycloak_realm_name']
+ },
+ {
+ 'key': 'KEYCLOAK_AUTH_SERVER_URL',
+ 'value': os.environ['keycloak_auth_server_url']
+ },
+ {
+ 'key': 'KEYCLOAK_CLIENT_NAME',
+ 'value': os.environ['keycloak_client_name']
+ },
+ {
+ 'key': 'KEYCLOAK_CLIENT_SECRET',
+ 'value': os.environ['keycloak_client_secret']
+ },
+ {
+ 'key': 'CONF_OS',
+ 'value': os.environ['conf_os_family']
+ },
+ {
+ 'key': 'SERVICE_BASE_NAME',
+ 'value': ssn_conf['service_base_name']
+ },
+ {
+ 'key': 'EDGE_INSTANCE_SIZE',
+ 'value': os.environ['azure_edge_instance_size']
+ },
+ {
+ 'key': 'SUBNET_ID',
+ 'value': ssn_conf['subnet_name']
+ },
+ {
+ 'key': 'REGION',
+ 'value': ssn_conf['region']
+ },
+ {
+ 'key': 'ZONE',
+ 'value': ''
+ },
+ {
+ 'key': 'TAG_RESOURCE_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'SG_IDS',
+ 'value': ssn_conf['security_group_name']
+ },
+ {
+ 'key': 'SSN_INSTANCE_SIZE',
+ 'value': os.environ['azure_ssn_instance_size']
+ },
+ {
+ 'key': 'VPC_ID',
+ 'value': ssn_conf['vpc_name']
+ },
+ {
+ 'key': 'CONF_KEY_DIR',
+ 'value': os.environ['conf_key_dir']
+ },
+ {
+ 'key': 'LDAP_HOST',
+ 'value': os.environ['ldap_host']
+ },
+ {
+ 'key': 'LDAP_DN',
+ 'value': os.environ['ldap_dn']
+ },
+ {
+ 'key': 'LDAP_OU',
+ 'value': os.environ['ldap_ou']
+ },
+ {
+ 'key': 'LDAP_USER',
+ 'value': os.environ['ldap_service_username']
+ },
+ {
+ 'key': 'LDAP_USER_PASSWORD',
+ 'value': os.environ['ldap_service_password']
+ },
+ {
+ 'key': 'AZURE_RESOURCE_GROUP_NAME',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_SSN_STORAGE_ACCOUNT_TAG',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_SHARED_STORAGE_ACCOUNT_TAG',
+ 'value': ''
+ },
+ {
+ 'key': 'SUBNET2_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'VPC2_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'PEERING_ID',
+ 'value': ''
}
+ ]
+
+ if os.environ['azure_datalake_enable'] == 'false':
+ cloud_params.append(
+ {
+ 'key': 'AZURE_DATALAKE_TAG',
+ 'value': ''
+ })
+ cloud_params.append(
+ {
+ 'key': 'AZURE_CLIENT_ID',
+ 'value': ''
+ })
if os.environ['azure_oauth2_enabled'] == 'false':
ldap_login = 'true'
tenant_id = json.dumps(AzureMeta().sp_creds['tenantId']).replace('"', '')
@@ -215,22 +319,16 @@ if __name__ == "__main__":
datalake_application_id = os.environ['azure_application_id']
datalake_store_name = None
else:
- mongo_parameters = {
- "azure_resource_group_name": ssn_conf['resource_group_name'],
- "azure_region": ssn_conf['region'],
- "azure_vpc_name": ssn_conf['vpc_name'],
- "azure_subnet_name": ssn_conf['subnet_name'],
- "conf_service_base_name": ssn_conf['service_base_name'],
- "azure_security_group_name": ssn_conf['security_group_name'],
- "conf_os_family": os.environ['conf_os_family'],
- "conf_key_dir": os.environ['conf_key_dir'],
- "ssn_instance_size": os.environ['azure_ssn_instance_size'],
- "edge_instance_size": os.environ['azure_edge_instance_size'],
- "ssn_storage_account_tag_name": ssn_conf['ssn_storage_account_name'],
- "shared_storage_account_tag_name": ssn_conf['shared_storage_account_name'],
- "datalake_tag_name": ssn_conf['datalake_store_name'],
- "azure_client_id": os.environ['azure_application_id']
- }
+ cloud_params.append(
+ {
+ 'key': 'AZURE_DATALAKE_TAG',
+ 'value': ssn_conf['datalake_store_name']
+ })
+ cloud_params.append(
+ {
+ 'key': 'AZURE_CLIENT_ID',
+ 'value': os.environ['azure_application_id']
+ })
tenant_id = json.dumps(AzureMeta().sp_creds['tenantId']).replace('"', '')
subscription_id = json.dumps(AzureMeta().sp_creds['subscriptionId']).replace('"', '')
datalake_application_id = os.environ['azure_application_id']
@@ -240,14 +338,14 @@ if __name__ == "__main__":
params = "--hostname {} --keyfile {} --dlab_path {} --os_user {} --os_family {} --request_id {} \
--resource {} --service_base_name {} --cloud_provider {} --billing_enabled {} --authentication_file {} \
--offer_number {} --currency {} --locale {} --region_info {} --ldap_login {} --tenant_id {} \
- --application_id {} --datalake_store_name {} --mongo_parameters '{}' --subscription_id {} \
+ --application_id {} --datalake_store_name {} --cloud_params '{}' --subscription_id {} \
--validate_permission_scope {}". \
format(ssn_conf['instnace_ip'], ssn_conf['ssh_key_path'], os.environ['ssn_dlab_path'],
ssn_conf['dlab_ssh_user'], os.environ['conf_os_family'], os.environ['request_id'],
os.environ['conf_resource'], ssn_conf['service_base_name'], os.environ['conf_cloud_provider'],
billing_enabled, azure_auth_path, os.environ['azure_offer_number'],
os.environ['azure_currency'], os.environ['azure_locale'], os.environ['azure_region_info'],
- ldap_login, tenant_id, datalake_application_id, datalake_store_name, json.dumps(mongo_parameters),
+ ldap_login, tenant_id, datalake_application_id, datalake_store_name, json.dumps(cloud_params),
subscription_id, os.environ['azure_validate_permission_scope'])
local("~/scripts/{}.py {}".format('configure_ui', params))
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index e4f7cf5..80fd3a7 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -28,6 +28,7 @@ import sys, os
from fabric.api import *
from dlab.ssn_lib import *
import traceback
+import json
if __name__ == "__main__":
local_log_filename = "{}_{}.log".format(os.environ['conf_resource'], os.environ['request_id'])
@@ -255,16 +256,126 @@ if __name__ == "__main__":
try:
logging.info('[CONFIGURE SSN INSTANCE UI]')
print('[CONFIGURE SSN INSTANCE UI]')
- mongo_parameters = {
- "conf_service_base_name": os.environ['conf_service_base_name'],
- "conf_os_family": os.environ['conf_os_family'],
- "conf_key_dir": os.environ['conf_key_dir']
- }
+
+ cloud_params = [
+ {
+ 'key': 'KEYCLOAK_REALM_NAME',
+ 'value': os.environ['keycloak_realm_name']
+ },
+ {
+ 'key': 'KEYCLOAK_AUTH_SERVER_URL',
+ 'value': os.environ['keycloak_auth_server_url']
+ },
+ {
+ 'key': 'KEYCLOAK_CLIENT_NAME',
+ 'value': os.environ['keycloak_client_name']
+ },
+ {
+ 'key': 'KEYCLOAK_CLIENT_SECRET',
+ 'value': os.environ['keycloak_client_secret']
+ },
+ {
+ 'key': 'CONF_OS',
+ 'value': os.environ['conf_os_family']
+ },
+ {
+ 'key': 'SERVICE_BASE_NAME',
+ 'value': os.environ['conf_service_base_name']
+ },
+ {
+ 'key': 'EDGE_INSTANCE_SIZE',
+ 'value': ''
+ },
+ {
+ 'key': 'SUBNET_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'REGION',
+ 'value': ''
+ },
+ {
+ 'key': 'ZONE',
+ 'value': ''
+ },
+ {
+ 'key': 'TAG_RESOURCE_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'SG_IDS',
+ 'value': ''
+ },
+ {
+ 'key': 'SSN_INSTANCE_SIZE',
+ 'value': ''
+ },
+ {
+ 'key': 'VPC_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'CONF_KEY_DIR',
+ 'value': os.environ['conf_key_dir']
+ },
+ {
+ 'key': 'LDAP_HOST',
+ 'value': os.environ['ldap_host']
+ },
+ {
+ 'key': 'LDAP_DN',
+ 'value': os.environ['ldap_dn']
+ },
+ {
+ 'key': 'LDAP_OU',
+ 'value': os.environ['ldap_ou']
+ },
+ {
+ 'key': 'LDAP_USER',
+ 'value': os.environ['ldap_service_username']
+ },
+ {
+ 'key': 'LDAP_USER_PASSWORD',
+ 'value': os.environ['ldap_service_password']
+ },
+ {
+ 'key': 'AZURE_RESOURCE_GROUP_NAME',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_SSN_STORAGE_ACCOUNT_TAG',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_SHARED_STORAGE_ACCOUNT_TAG',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_DATALAKE_TAG',
+ 'value': ''
+ },
+ {
+ 'key': 'AZURE_CLIENT_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'SUBNET2_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'VPC2_ID',
+ 'value': ''
+ },
+ {
+ 'key': 'PEERING_ID',
+ 'value': ''
+ }
+ ]
params = "--hostname {} --keyfile {} --dlab_path {} --os_user {} --os_family {} --request_id {} \
- --resource {} --service_base_name {} --cloud_provider {} --mongo_parameters '{}'". \
+ --resource {} --service_base_name {} --cloud_provider {} --cloud_params '{}'". \
format(instance_hostname, ssn_conf['ssh_key_path'], os.environ['ssn_dlab_path'], ssn_conf['dlab_ssh_user'],
os.environ['conf_os_family'], os.environ['request_id'], os.environ['conf_resource'],
- ssn_conf['service_base_name'], os.environ['conf_cloud_provider'], json.dumps(mongo_parameters))
+ ssn_conf['service_base_name'], os.environ['conf_cloud_provider'], json.dumps(cloud_params))
try:
local("~/scripts/{}.py {}".format('configure_ui', params))
except:
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
index 02f3752..210dc90 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
@@ -56,10 +56,6 @@ parser.add_argument('--usage_type', type=str, default='', help='Column name in r
parser.add_argument('--usage', type=str, default='', help='Column name in report file that contains usage tag')
parser.add_argument('--cost', type=str, default='', help='Column name in report file that contains cost tag')
parser.add_argument('--resource_id', type=str, default='', help='Column name in report file that contains dlab resource id tag')
-parser.add_argument('--keycloak_realm_name', type=str, default='')
-parser.add_argument('--keycloak_auth_server_url', type=str, default='')
-parser.add_argument('--keycloak_client_name', type=str, default='')
-parser.add_argument('--keycloak_client_secret', type=str, default='')
parser.add_argument('--tags', type=str, default='', help='Column name in report file that contains tags')
args = parser.parse_args()
@@ -115,10 +111,6 @@ def yml_self_service(path):
config_orig = config_yml_r.read()
config_orig = config_orig.replace('billingSchedulerEnabled: false', 'billingSchedulerEnabled: true')
- config_orig = config_orig.replace('KEYCLOAK_REALM_NAME', args.keycloak_realm_name)
- config_orig = config_orig.replace('KEYCLOAK_AUTH_SERVER_URL', args.keycloak_auth_server_url)
- config_orig = config_orig.replace('KEYCLOAK_CLIENT_NAME', args.keycloak_client_name)
- config_orig = config_orig.replace('KEYCLOAK_CLIENT_SECRET', args.keycloak_client_secret)
f = open(path, 'w')
f.write(config_orig)
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py
index a7a5fc8..2890a51 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_mongo.py
@@ -33,7 +33,7 @@ outfile = "/etc/mongo_params.yml"
parser = argparse.ArgumentParser()
parser.add_argument('--dlab_path', type=str, default='')
-parser.add_argument('--mongo_parameters', type=str, default='')
+#parser.add_argument('--mongo_parameters', type=str, default='')
args = parser.parse_args()
@@ -74,7 +74,7 @@ if __name__ == "__main__":
mongo_passwd = "PASSWORD"
mongo_ip = read_yml_conf(path,'net','bindIp')
mongo_port = read_yml_conf(path,'net','port')
- mongo_parameters = json.loads(args.mongo_parameters)
+ #mongo_parameters = json.loads(args.mongo_parameters)
# Setting up admin's password and enabling security
client = MongoClient(mongo_ip + ':' + str(mongo_port))
pass_upd = True
@@ -84,12 +84,12 @@ if __name__ == "__main__":
time.sleep(5)
client.dlabdb.add_user('admin', mongo_passwd, roles=[{'role':'userAdminAnyDatabase','db':'admin'}])
client.dlabdb.command('grantRolesToUser', "admin", roles=["readWrite"])
- set_mongo_parameters(client, mongo_parameters)
- with open(args.dlab_path + 'tmp/mongo_roles.json', 'r') as data:
- json_data = json.load(data)
- for i in json_data:
- client.dlabdb.roles.insert_one(i)
- client.dlabdb.security.create_index("expireAt", expireAfterSeconds=7200)
+ # set_mongo_parameters(client, mongo_parameters)
+ # with open(args.dlab_path + 'tmp/mongo_roles.json', 'r') as data:
+ # json_data = json.load(data)
+ # for i in json_data:
+ # client.dlabdb.roles.insert_one(i)
+ # client.dlabdb.security.create_index("expireAt", expireAfterSeconds=7200)
if add_2_yml_config(path,'security','authorization','enabled'):
command = ['service', 'mongod', 'restart']
subprocess.call(command, shell=False)
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index 23b4329..c56b434 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -61,7 +61,7 @@ parser.add_argument('--application_id', type=str, default=None)
parser.add_argument('--subscription_id', type=str, default=None)
parser.add_argument('--datalake_store_name', type=str, default=None)
parser.add_argument('--validate_permission_scope', type=str, default=None)
-parser.add_argument('--mongo_parameters', type=str, default='')
+parser.add_argument('--cloud_params', type=str, default='')
parser.add_argument('--dlab_id', type=str, default=None)
parser.add_argument('--usage_date', type=str, default=None)
parser.add_argument('--product', type=str, default=None)
@@ -122,9 +122,8 @@ def configure_mongo(mongo_passwd):
args.cloud_provider,
env.host_string))
sudo('mv /tmp/mongo_roles.json ' + args.dlab_path + 'tmp/')
- mongo_parameters = json.loads(args.mongo_parameters)
- sudo("python " + args.dlab_path + "tmp/configure_mongo.py --dlab_path {} --mongo_parameters '{}'".format(
- args.dlab_path, json.dumps(mongo_parameters)))
+ sudo("python " + args.dlab_path + "tmp/configure_mongo.py --dlab_path {} ".format(
+ args.dlab_path))
except Exception as err:
traceback.print_exc()
print('Failed to configure MongoDB: ', str(err))
@@ -150,7 +149,7 @@ def build_ui():
sudo('/opt/maven/bin/mvn -P{} -DskipTests package'.format(args.cloud_provider))
sudo('mkdir -p {}/webapp/'.format(args.dlab_path))
- for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
+ for service in ['self-service', 'provisioning-service', 'billing']:
sudo('mkdir -p {}/webapp/{}/lib/'.format(args.dlab_path, service))
sudo('mkdir -p {}/webapp/{}/conf/'.format(args.dlab_path, service))
sudo('cp {0}/sources/services/self-service/self-service.yml {0}/webapp/self-service/conf/'.format(
@@ -162,21 +161,6 @@ def build_ui():
sudo('cp {0}/sources/services/provisioning-service/target/provisioning-service-*.jar '
'{0}/webapp/provisioning-service/lib/'.format(args.dlab_path))
- sudo('sed -i "s/LDAP_HOST/{0}/g" {1}/sources/services/security-service/security.yml'.format(
- os.environ['ldap_hostname'], args.dlab_path))
- sudo('sed -i "s/LDAP_USER/{0}/g" {1}/sources/services/security-service/security.yml'.format(
- os.environ['ldap_service_username'], args.dlab_path))
- sudo('sed -i "s/LDAP_DN/{0}/g" {1}/sources/services/security-service/security.yml'.format(os.environ['ldap_dn'],
- args.dlab_path))
- sudo('sed -i "s/LDAP_OU/{0}/g" {1}/sources/services/security-service/security.yml'.format(os.environ['ldap_ou'],
- args.dlab_path))
- sudo("sed -i 's/LDAP_PASS/{0}/g' {1}/sources/services/security-service/security.yml".format(
- os.environ['ldap_service_password'], args.dlab_path))
- sudo('cp {0}/sources/services/security-service/security.yml {0}/webapp/security-service/conf/'.format(
- args.dlab_path))
- sudo('cp {0}/sources/services/security-service/target/security-service-*.jar '
- '{0}/webapp/security-service/lib/'.format(args.dlab_path))
-
if args.cloud_provider == 'azure':
sudo('cp {0}/sources/services/billing-azure/billing.yml {0}/webapp/billing/conf/'.format(args.dlab_path))
sudo('cp {0}/sources/services/billing-azure/target/billing-azure*.jar {0}/webapp/billing/lib/'.format(
@@ -235,7 +219,7 @@ if __name__ == "__main__":
start_ss(args.keyfile, env.host_string, dlab_conf_dir, web_path,
args.os_user, mongo_passwd, keystore_passwd, args.cloud_provider,
args.service_base_name, args.tag_resource_id, args.billing_tag, args.account_id,
- args.billing_bucket, args.aws_job_enabled, args.dlab_path, args.billing_enabled,
+ args.billing_bucket, args.aws_job_enabled, args.dlab_path, args.billing_enabled, args.cloud_params,
args.authentication_file, args.offer_number, args.currency, args.locale,
args.region_info, args.ldap_login, args.tenant_id, args.application_id,
args.hostname, args.datalake_store_name, args.subscription_id, args.validate_permission_scope,
diff --git a/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf b/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
index f835067..c343cb7 100644
--- a/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
+++ b/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
@@ -34,15 +34,15 @@ stdout_logfile=/var/log/application/ui.log
redirect_stderr=true
environment=DLAB_CONF_DIR="WEB_CONF"
-[program:secserv]
-command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 security-service/security-service.jar server WEB_CONFsecurity.yml
-directory=WEB_APP_DIR
-autorestart=true
-priority=20
-user=OS_USR
-stdout_logfile=/var/log/application/security-service.log
-redirect_stderr=true
-environment=DLAB_CONF_DIR="WEB_CONF"
+; [program:secserv]
+; command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 security-service/security-service.jar server WEB_CONFsecurity.yml
+; directory=WEB_APP_DIR
+; autorestart=true
+; priority=20
+; user=OS_USR
+; stdout_logfile=/var/log/application/security-service.log
+; redirect_stderr=true
+; environment=DLAB_CONF_DIR="WEB_CONF"
[program:provserv]
command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 provisioning-service/provisioning-service.jar server WEB_CONFprovisioning.yml
diff --git a/services/self-service/self-service.yml b/services/self-service/self-service.yml
index 22280fd..aa0d0fa 100644
--- a/services/self-service/self-service.yml
+++ b/services/self-service/self-service.yml
@@ -195,24 +195,30 @@ jerseyClient:
chunkedEncodingEnabled: true
cloudProperties:
- os: debian
- serviceBaseName: dev
- edgeInstanceSize: t2.medium
- subnetId: subnet-22db937a
- region: us-west-2
- zone: us
- confTagResourceId: user:tag
- securityGroupIds: sg-4d42dc35,sg-f19a0389,sg-71e27b09,sg-d3e67fab
- ssnInstanceSize: t2.large
- notebookVpcId: vpc-83c469e4
- notebookSubnetId: subnet-22db937a
- confKeyDir: /root/keys/
- vpcId: vpc-83c469e4
+ os: CONF_OS
+ serviceBaseName: SERVICE_BASE_NAME
+ edgeInstanceSize: EDGE_INSTANCE_SIZE
+ subnetId: SUBNET_ID
+ region: REGION
+ zone: ZONE
+ confTagResourceId: TAG_RESOURCE_ID
+ securityGroupIds: SG_IDS
+ ssnInstanceSize: SSN_INSTANCE_SIZE
+ notebookVpcId: VPC2_ID
+ notebookSubnetId: SUBNET2_ID
+ confKeyDir: CONF_KEY_DIR
+ vpcId: VPC_ID
+ peeringId: PEERING_ID
+ azureResourceGroupName: AZURE_RESOURCE_GROUP_NAME
+ ssnStorageAccountTagName: AZURE_SSN_STORAGE_ACCOUNT_TAG
+ sharedStorageAccountTagName: AZURE_SHARED_STORAGE_ACCOUNT_TAG
+ datalakeTagName: AZURE_DATALAKE_TAG
+ azureClientId: AZURE_CLIENT_ID
ldap:
- host: ec2-52-36-63-161.us-west-2.compute.amazonaws.com
- dn: dc=example,dc=com
- ou: ou=People
- user: cn=admin
- password: pass
+ host: LDAP_HOST
+ dn: LDAP_DN
+ ou: LDAP_OU
+ user: LDAP_USER
+ password: LDAP_USER_PASSWORD
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org