You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by ad...@apache.org on 2019/09/11 09:39:58 UTC
[incubator-dlab] 01/01: Devops gcp billing
This is an automated email from the ASF dual-hosted git repository.
adamsd pushed a commit to branch bil-devops
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit c012d210fe600777b58e9a5a521b46ad2aa7783e
Author: AdamsDisturber <ad...@gmail.com>
AuthorDate: Wed Sep 11 12:36:18 2019 +0300
Devops gcp billing
---
.../src/general/lib/os/debian/ssn_lib.py | 25 ++++++--
.../src/general/lib/os/redhat/ssn_lib.py | 74 ++++++++++++++++++++--
.../src/ssn/scripts/configure_billing.py | 4 ++
.../src/ssn/scripts/configure_ui.py | 7 +-
.../src/ssn/templates/supervisor_svc.conf | 12 +++-
.../webapp/src/dictionary/global.dictionary.ts | 4 +-
6 files changed, 109 insertions(+), 17 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index 637d014..a4d24cc 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -190,11 +190,22 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
- with open('/root/templates/supervisor_svc.conf', 'r') as f:
- text = f.read()
- text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)
- with open('/root/templates/supervisor_svc.conf', 'w') as f:
- f.write(text)
+ if cloud_provider == 'gcp':
+ conf_parameter_name = '--spring.config.location='
+ with open('/root/templates/supervisor_svc.conf', 'r') as f:
+ text = f.read()
+ text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
+ .replace('CONF_PARAMETER_NAME', conf_parameter_name)
+ with open('/root/templates/supervisor_svc.conf', 'w') as f:
+ f.write(text)
+ elif cloud_provider == 'aws' or 'azure':
+ conf_parameter_name = '--conf '
+ with open('/root/templates/supervisor_svc.conf', 'r') as f:
+ text = f.read()
+ text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
+ .replace('CONF_PARAMETER_NAME', conf_parameter_name)
+ with open('/root/templates/supervisor_svc.conf', 'w') as f:
+ f.write(text)
put('/root/templates/supervisor_svc.conf', '/tmp/supervisor_svc.conf')
sudo('mv /tmp/supervisor_svc.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
sudo('cp ' + os.environ['ssn_dlab_path'] +
@@ -204,7 +215,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
try:
sudo('mkdir -p /var/log/application')
run('mkdir -p /tmp/yml_tmp/')
- for service in ['self-service', 'security-service', 'provisioning-service']:
+ for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
jar = sudo('cd {0}{1}/lib/; find {1}*.jar -type f'.format(web_path, service))
sudo('ln -s {0}{2}/lib/{1} {0}{2}/{2}.jar '.format(web_path, jar, service))
sudo('cp {0}/webapp/{1}/conf/*.yml /tmp/yml_tmp/'.format(dlab_path, service))
@@ -283,7 +294,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
cost,
resource_id,
tags)
- #sudo('python /tmp/configure_billing.py {}'.format(params))
+ sudo('python /tmp/configure_billing.py {}'.format(params))
try:
sudo('keytool -genkeypair -alias dlab -keyalg RSA -validity 730 -storepass {1} -keypass {1} \
-keystore /home/{0}/keys/dlab.keystore.jks -keysize 2048 -dname "CN=localhost"'.format(os_user, keystore_passwd))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index cdcafee..b99d78d 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -216,11 +216,22 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
- with open('/root/templates/supervisor_svc.conf', 'r') as f:
- text = f.read()
- text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)
- with open('/root/templates/supervisor_svc.ini', 'w') as f:
- f.write(text)
+ if cloud_provider == 'gcp':
+ conf_parameter_name = '--spring.config.location='
+ with open('/root/templates/supervisor_svc.conf', 'r') as f:
+ text = f.read()
+ text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
+ .replace('CONF_PARAMETER_NAME', conf_parameter_name)
+ with open('/root/templates/supervisor_svc.conf', 'w') as f:
+ f.write(text)
+ elif cloud_provider == 'aws' or 'azure':
+ conf_parameter_name = '--conf '
+ with open('/root/templates/supervisor_svc.conf', 'r') as f:
+ text = f.read()
+ text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
+ .replace('CONF_PARAMETER_NAME', conf_parameter_name)
+ with open('/root/templates/supervisor_svc.conf', 'w') as f:
+ f.write(text)
put('/root/templates/supervisor_svc.ini', '/tmp/supervisor_svc.ini')
sudo('mv /tmp/supervisor_svc.ini ' + os.environ['ssn_dlab_path'] + 'tmp/')
sudo('cp ' + os.environ['ssn_dlab_path'] +
@@ -230,7 +241,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
try:
sudo('mkdir -p /var/log/application')
run('mkdir -p /tmp/yml_tmp/')
- for service in ['self-service', 'security-service', 'provisioning-service']:
+ for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
jar = sudo('cd {0}{1}/lib/; find {1}*.jar -type f'.format(web_path, service))
sudo('ln -s {0}{2}/lib/{1} {0}{2}/{2}.jar '.format(web_path, jar, service))
sudo('cp {0}/webapp/{1}/conf/*.yml /tmp/yml_tmp/'.format(dlab_path, service))
@@ -264,6 +275,57 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
append_result("Unable to upload webapp jars. ", str(err))
sys.exit(1)
+ if billing_enabled:
+ local('scp -i {} /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile,
+ host_string))
+ params = '--cloud_provider {} ' \
+ '--infrastructure_tag {} ' \
+ '--tag_resource_id {} ' \
+ '--billing_tag {} ' \
+ '--account_id {} ' \
+ '--billing_bucket {} ' \
+ '--aws_job_enabled {} ' \
+ '--report_path "{}" ' \
+ '--mongo_password {} ' \
+ '--dlab_dir {} ' \
+ '--authentication_file "{}" ' \
+ '--offer_number {} ' \
+ '--currency {} ' \
+ '--locale {} ' \
+ '--region_info {} ' \
+ '--dlab_id {} ' \
+ '--usage_date {} ' \
+ '--product {} ' \
+ '--usage_type {} ' \
+ '--usage {} ' \
+ '--cost {} ' \
+ '--resource_id {} ' \
+ '--tags {}'.\
+ format(cloud_provider,
+ service_base_name,
+ tag_resource_id,
+ billing_tag,
+ account_id,
+ billing_bucket,
+ aws_job_enabled,
+ report_path,
+ mongo_passwd,
+ dlab_path,
+ authentication_file,
+ offer_number,
+ currency,
+ locale,
+ region_info,
+ dlab_id,
+ usage_date,
+ product,
+ usage_type,
+ usage,
+ cost,
+ resource_id,
+ tags)
+ sudo('python /tmp/configure_billing.py {}'.format(params))
+
try:
sudo('keytool -genkeypair -alias dlab -keyalg RSA -validity 730 -storepass {1} -keypass {1} \
-keystore /home/{0}/keys/dlab.keystore.jks -keysize 2048 -dname "CN=localhost"'.format(
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
index 210dc90..05fd4f2 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
@@ -97,6 +97,10 @@ def yml_billing(path):
config_orig = config_orig.replace('<LOCALE>', args.locale)
config_orig = config_orig.replace('<REGION_INFO>', args.region_info)
config_orig = config_orig.replace('<MONGODB_PASSWORD>', args.mongo_password)
+ elif args.cloud_provider == 'gcp':
+ config_orig = config_orig.replace('<CONF_SERVICE_BASE_NAME>', args.infrastructure_tag)
+ config_orig = config_orig.replace('<MONGODB_PASSWORD>', args.mongo_password)
+ config_orig = config_orig.replace('<BILLING_DATASET_NAME>', args.billing_dataset_name)
f = open(path, 'w')
f.write(config_orig)
f.close()
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index cc4453a..23b4329 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -150,7 +150,7 @@ def build_ui():
sudo('/opt/maven/bin/mvn -P{} -DskipTests package'.format(args.cloud_provider))
sudo('mkdir -p {}/webapp/'.format(args.dlab_path))
- for service in ['self-service', 'security-service', 'provisioning-service']:
+ for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
sudo('mkdir -p {}/webapp/{}/lib/'.format(args.dlab_path, service))
sudo('mkdir -p {}/webapp/{}/conf/'.format(args.dlab_path, service))
sudo('cp {0}/sources/services/self-service/self-service.yml {0}/webapp/self-service/conf/'.format(
@@ -186,6 +186,11 @@ def build_ui():
sudo(
'cp {0}/sources/services/billing-aws/target/billing-aws*.jar {0}/webapp/billing/lib/'.format(
args.dlab_path))
+ elif args.cloud_provider == 'gcp':
+ sudo('cp {0}/sources/services/billing-gcp/billing.yml {0}/webapp/billing/conf/'.format(args.dlab_path))
+ sudo(
+ 'cp {0}/sources/services/billing-gcp/target/billing-gcp*.jar {0}/webapp/billing/lib/'.format(
+ args.dlab_path))
except Exception as err:
traceback.print_exc()
print('Failed to build UI: ', str(err))
diff --git a/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf b/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
index db94bd6..2851338 100644
--- a/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
+++ b/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
@@ -52,4 +52,14 @@ priority=20
user=OS_USR
stdout_logfile=/var/log/application/provision-service.log
redirect_stderr=true
-environment=DLAB_CONF_DIR="WEB_CONF"
\ No newline at end of file
+environment=DLAB_CONF_DIR="WEB_CONF"
+
+[program:billing]
+command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 billing/billing.jar CONF_PARAMETER_NAMEWEB_CONFbilling.yml
+directory=WEB_APP_DIR
+autorestart=true
+priority=20
+user=OS_USR
+stdout_logfile=/var/log/application/billing.log
+redirect_stderr=true
+environment=DLAB_CONF_DIR="WEB_CONF"
diff --git a/services/self-service/src/main/resources/webapp/src/dictionary/global.dictionary.ts b/services/self-service/src/main/resources/webapp/src/dictionary/global.dictionary.ts
index 31b86c6..6d0d8ea 100644
--- a/services/self-service/src/main/resources/webapp/src/dictionary/global.dictionary.ts
+++ b/services/self-service/src/main/resources/webapp/src/dictionary/global.dictionary.ts
@@ -19,8 +19,8 @@
// from './{{ aws | gcp | azure }}.dictionary';
-import { NAMING_CONVENTION } from './gcp.dictionary';
+import { NAMING_CONVENTION } from './CLOUD_PROVIDER.dictionary';
-export * from './gcp.dictionary';
+export * from './CLOUD_PROVIDER.dictionary';
export const DICTIONARY = Object.freeze(NAMING_CONVENTION);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org