You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by ad...@apache.org on 2019/09/11 09:39:57 UTC
[incubator-dlab] branch bil-devops created (now c012d21)
This is an automated email from the ASF dual-hosted git repository.
adamsd pushed a change to branch bil-devops
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at c012d21 Devops gcp billing
This branch includes the following new commits:
new c012d21 Devops gcp billing
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: Devops gcp billing
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adamsd pushed a commit to branch bil-devops
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit c012d210fe600777b58e9a5a521b46ad2aa7783e
Author: AdamsDisturber <ad...@gmail.com>
AuthorDate: Wed Sep 11 12:36:18 2019 +0300
Devops gcp billing
---
.../src/general/lib/os/debian/ssn_lib.py | 25 ++++++--
.../src/general/lib/os/redhat/ssn_lib.py | 74 ++++++++++++++++++++--
.../src/ssn/scripts/configure_billing.py | 4 ++
.../src/ssn/scripts/configure_ui.py | 7 +-
.../src/ssn/templates/supervisor_svc.conf | 12 +++-
.../webapp/src/dictionary/global.dictionary.ts | 4 +-
6 files changed, 109 insertions(+), 17 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index 637d014..a4d24cc 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -190,11 +190,22 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
- with open('/root/templates/supervisor_svc.conf', 'r') as f:
- text = f.read()
- text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)
- with open('/root/templates/supervisor_svc.conf', 'w') as f:
- f.write(text)
+ if cloud_provider == 'gcp':
+ conf_parameter_name = '--spring.config.location='
+ with open('/root/templates/supervisor_svc.conf', 'r') as f:
+ text = f.read()
+ text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
+ .replace('CONF_PARAMETER_NAME', conf_parameter_name)
+ with open('/root/templates/supervisor_svc.conf', 'w') as f:
+ f.write(text)
+ elif cloud_provider == 'aws' or 'azure':
+ conf_parameter_name = '--conf '
+ with open('/root/templates/supervisor_svc.conf', 'r') as f:
+ text = f.read()
+ text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
+ .replace('CONF_PARAMETER_NAME', conf_parameter_name)
+ with open('/root/templates/supervisor_svc.conf', 'w') as f:
+ f.write(text)
put('/root/templates/supervisor_svc.conf', '/tmp/supervisor_svc.conf')
sudo('mv /tmp/supervisor_svc.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
sudo('cp ' + os.environ['ssn_dlab_path'] +
@@ -204,7 +215,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
try:
sudo('mkdir -p /var/log/application')
run('mkdir -p /tmp/yml_tmp/')
- for service in ['self-service', 'security-service', 'provisioning-service']:
+ for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
jar = sudo('cd {0}{1}/lib/; find {1}*.jar -type f'.format(web_path, service))
sudo('ln -s {0}{2}/lib/{1} {0}{2}/{2}.jar '.format(web_path, jar, service))
sudo('cp {0}/webapp/{1}/conf/*.yml /tmp/yml_tmp/'.format(dlab_path, service))
@@ -283,7 +294,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
cost,
resource_id,
tags)
- #sudo('python /tmp/configure_billing.py {}'.format(params))
+ sudo('python /tmp/configure_billing.py {}'.format(params))
try:
sudo('keytool -genkeypair -alias dlab -keyalg RSA -validity 730 -storepass {1} -keypass {1} \
-keystore /home/{0}/keys/dlab.keystore.jks -keysize 2048 -dname "CN=localhost"'.format(os_user, keystore_passwd))
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index cdcafee..b99d78d 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@ -216,11 +216,22 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
- with open('/root/templates/supervisor_svc.conf', 'r') as f:
- text = f.read()
- text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)
- with open('/root/templates/supervisor_svc.ini', 'w') as f:
- f.write(text)
+ if cloud_provider == 'gcp':
+ conf_parameter_name = '--spring.config.location='
+ with open('/root/templates/supervisor_svc.conf', 'r') as f:
+ text = f.read()
+ text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
+ .replace('CONF_PARAMETER_NAME', conf_parameter_name)
+ with open('/root/templates/supervisor_svc.conf', 'w') as f:
+ f.write(text)
+ elif cloud_provider == 'aws' or 'azure':
+ conf_parameter_name = '--conf '
+ with open('/root/templates/supervisor_svc.conf', 'r') as f:
+ text = f.read()
+ text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
+ .replace('CONF_PARAMETER_NAME', conf_parameter_name)
+ with open('/root/templates/supervisor_svc.conf', 'w') as f:
+ f.write(text)
put('/root/templates/supervisor_svc.ini', '/tmp/supervisor_svc.ini')
sudo('mv /tmp/supervisor_svc.ini ' + os.environ['ssn_dlab_path'] + 'tmp/')
sudo('cp ' + os.environ['ssn_dlab_path'] +
@@ -230,7 +241,7 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
try:
sudo('mkdir -p /var/log/application')
run('mkdir -p /tmp/yml_tmp/')
- for service in ['self-service', 'security-service', 'provisioning-service']:
+ for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
jar = sudo('cd {0}{1}/lib/; find {1}*.jar -type f'.format(web_path, service))
sudo('ln -s {0}{2}/lib/{1} {0}{2}/{2}.jar '.format(web_path, jar, service))
sudo('cp {0}/webapp/{1}/conf/*.yml /tmp/yml_tmp/'.format(dlab_path, service))
@@ -264,6 +275,57 @@ def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
append_result("Unable to upload webapp jars. ", str(err))
sys.exit(1)
+ if billing_enabled:
+ local('scp -i {} /root/scripts/configure_billing.py {}:/tmp/configure_billing.py'.format(keyfile,
+ host_string))
+ params = '--cloud_provider {} ' \
+ '--infrastructure_tag {} ' \
+ '--tag_resource_id {} ' \
+ '--billing_tag {} ' \
+ '--account_id {} ' \
+ '--billing_bucket {} ' \
+ '--aws_job_enabled {} ' \
+ '--report_path "{}" ' \
+ '--mongo_password {} ' \
+ '--dlab_dir {} ' \
+ '--authentication_file "{}" ' \
+ '--offer_number {} ' \
+ '--currency {} ' \
+ '--locale {} ' \
+ '--region_info {} ' \
+ '--dlab_id {} ' \
+ '--usage_date {} ' \
+ '--product {} ' \
+ '--usage_type {} ' \
+ '--usage {} ' \
+ '--cost {} ' \
+ '--resource_id {} ' \
+ '--tags {}'.\
+ format(cloud_provider,
+ service_base_name,
+ tag_resource_id,
+ billing_tag,
+ account_id,
+ billing_bucket,
+ aws_job_enabled,
+ report_path,
+ mongo_passwd,
+ dlab_path,
+ authentication_file,
+ offer_number,
+ currency,
+ locale,
+ region_info,
+ dlab_id,
+ usage_date,
+ product,
+ usage_type,
+ usage,
+ cost,
+ resource_id,
+ tags)
+ sudo('python /tmp/configure_billing.py {}'.format(params))
+
try:
sudo('keytool -genkeypair -alias dlab -keyalg RSA -validity 730 -storepass {1} -keypass {1} \
-keystore /home/{0}/keys/dlab.keystore.jks -keysize 2048 -dname "CN=localhost"'.format(
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
index 210dc90..05fd4f2 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_billing.py
@@ -97,6 +97,10 @@ def yml_billing(path):
config_orig = config_orig.replace('<LOCALE>', args.locale)
config_orig = config_orig.replace('<REGION_INFO>', args.region_info)
config_orig = config_orig.replace('<MONGODB_PASSWORD>', args.mongo_password)
+ elif args.cloud_provider == 'gcp':
+ config_orig = config_orig.replace('<CONF_SERVICE_BASE_NAME>', args.infrastructure_tag)
+ config_orig = config_orig.replace('<MONGODB_PASSWORD>', args.mongo_password)
+ config_orig = config_orig.replace('<BILLING_DATASET_NAME>', args.billing_dataset_name)
f = open(path, 'w')
f.write(config_orig)
f.close()
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index cc4453a..23b4329 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -150,7 +150,7 @@ def build_ui():
sudo('/opt/maven/bin/mvn -P{} -DskipTests package'.format(args.cloud_provider))
sudo('mkdir -p {}/webapp/'.format(args.dlab_path))
- for service in ['self-service', 'security-service', 'provisioning-service']:
+ for service in ['self-service', 'security-service', 'provisioning-service', 'billing']:
sudo('mkdir -p {}/webapp/{}/lib/'.format(args.dlab_path, service))
sudo('mkdir -p {}/webapp/{}/conf/'.format(args.dlab_path, service))
sudo('cp {0}/sources/services/self-service/self-service.yml {0}/webapp/self-service/conf/'.format(
@@ -186,6 +186,11 @@ def build_ui():
sudo(
'cp {0}/sources/services/billing-aws/target/billing-aws*.jar {0}/webapp/billing/lib/'.format(
args.dlab_path))
+ elif args.cloud_provider == 'gcp':
+ sudo('cp {0}/sources/services/billing-gcp/billing.yml {0}/webapp/billing/conf/'.format(args.dlab_path))
+ sudo(
+ 'cp {0}/sources/services/billing-gcp/target/billing-gcp*.jar {0}/webapp/billing/lib/'.format(
+ args.dlab_path))
except Exception as err:
traceback.print_exc()
print('Failed to build UI: ', str(err))
diff --git a/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf b/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
index db94bd6..2851338 100644
--- a/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
+++ b/infrastructure-provisioning/src/ssn/templates/supervisor_svc.conf
@@ -52,4 +52,14 @@ priority=20
user=OS_USR
stdout_logfile=/var/log/application/provision-service.log
redirect_stderr=true
-environment=DLAB_CONF_DIR="WEB_CONF"
\ No newline at end of file
+environment=DLAB_CONF_DIR="WEB_CONF"
+
+[program:billing]
+command=java -Xmx1024M -jar -Duser.timezone=UTC -Dfile.encoding=UTF-8 billing/billing.jar CONF_PARAMETER_NAMEWEB_CONFbilling.yml
+directory=WEB_APP_DIR
+autorestart=true
+priority=20
+user=OS_USR
+stdout_logfile=/var/log/application/billing.log
+redirect_stderr=true
+environment=DLAB_CONF_DIR="WEB_CONF"
diff --git a/services/self-service/src/main/resources/webapp/src/dictionary/global.dictionary.ts b/services/self-service/src/main/resources/webapp/src/dictionary/global.dictionary.ts
index 31b86c6..6d0d8ea 100644
--- a/services/self-service/src/main/resources/webapp/src/dictionary/global.dictionary.ts
+++ b/services/self-service/src/main/resources/webapp/src/dictionary/global.dictionary.ts
@@ -19,8 +19,8 @@
// from './{{ aws | gcp | azure }}.dictionary';
-import { NAMING_CONVENTION } from './gcp.dictionary';
+import { NAMING_CONVENTION } from './CLOUD_PROVIDER.dictionary';
-export * from './gcp.dictionary';
+export * from './CLOUD_PROVIDER.dictionary';
export const DICTIONARY = Object.freeze(NAMING_CONVENTION);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org