You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/08/16 11:20:28 UTC
[incubator-dlab] branch develop updated: [GCP]: image version
changed [GCP]: project termination fixed [AWS]: Add endpoint_tag for SG for
EMR [GCP]: Dataproc creation fixed
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/develop by this push:
new 032eb39 [GCP]: image version changed [GCP]: project termination fixed [AWS]: Add endpoint_tag for SG for EMR [GCP]: Dataproc creation fixed
new 5e6032a Merge pull request #260 from bodnarmykola/DLAB-748
032eb39 is described below
commit 032eb390ee2673d4120a02439d99d6db6d493121
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Fri Aug 16 14:16:43 2019 +0300
[GCP]: image version changed
[GCP]: project termination fixed
[AWS]: Add endpoint_tag for SG for EMR
[GCP]: Dataproc creation fixed
---
infrastructure-provisioning/src/general/conf/dlab.ini | 2 +-
.../src/general/scripts/aws/dataengine-service_create.py | 9 ---------
.../src/general/scripts/aws/dataengine-service_prepare.py | 8 ++++++++
.../src/general/scripts/gcp/dataengine-service_create.py | 2 +-
.../src/general/scripts/gcp/project_terminate.py | 4 ++--
5 files changed, 12 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 5fcda67..54032ea 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -171,7 +171,7 @@ region = us-west1
### GCP zone name for whole dlab provisioning
zone = us-west1-a
### GCP ami name based on debian conf_os_family for all dlab instances
-debian_image_name = /projects/ubuntu-os-cloud/global/images/ubuntu-1604-xenial-v20190628
+debian_image_name = /projects/ubuntu-os-cloud/global/images/ubuntu-1604-xenial-v20190807
### GCP ami name based on RedHat conf_os_family for all dlab instances
redhat_image_name =
### Prefix of the private subnet
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
index 4fe5fb7..8a61270 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
@@ -306,15 +306,6 @@ def build_emr_cluster(args):
'Value': '{}:{}'.format(args.service_base_name, args.name)})
tags.append({'Key': os.environ['conf_billing_tag_key'],
'Value': os.environ['conf_billing_tag_value']})
- if 'conf_additional_tags' in os.environ:
- for tag in os.environ['conf_additional_tags'].split(';'):
- tags.append(
- {
- 'Key': tag.split(':')[0],
- 'Value': tag.split(':')[1]
- }
- )
-
prefix = "jars/" + args.release_label + "/lib/"
jars_exist = get_object_count(args.s3_bucket, prefix)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
index 20e72f6..61963d3 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
@@ -81,6 +81,8 @@ if __name__ == "__main__":
emr_conf['tag_name'] = '{0}-Tag'.format(emr_conf['service_base_name'])
emr_conf['key_name'] = os.environ['conf_key_name']
+ emr_conf['endpoint_tag'] = os.environ['endpoint_name']
+ emr_conf['project_tag'] = os.environ['project_name']
emr_conf['region'] = os.environ['aws_region']
emr_conf['release_label'] = os.environ['emr_version']
emr_conf['edge_instance_name'] = '{0}-{1}-edge'.format(emr_conf['service_base_name'], os.environ['project_name'])
@@ -229,6 +231,12 @@ if __name__ == "__main__":
emr_conf['service_base_name'],
emr_conf['cluster_name'], True)
try:
+ if 'conf_additional_tags' in os.environ:
+ os.environ['conf_additional_tags'] = os.environ['conf_additional_tags'] + ';project_tag:{0};endpoint_tag:{1};'.format(
+ emr_conf['project_tag'], emr_conf['endpoint_tag'])
+ else:
+ os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(emr_conf['project_tag'], emr_conf['endpoint_tag'])
+ print('Additional tags will be added: {}'.format(os.environ['conf_additional_tags']))
local("~/scripts/{}.py {}".format('common_create_security_group', params))
except:
traceback.print_exc()
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
index f39e528..004a1c9 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
@@ -57,7 +57,7 @@ def send_parser_job(args, cluster_name, cluster_version):
job_body['job']['placement']['clusterName'] = cluster_name
job_body['job']['pysparkJob']['mainPythonFileUri'] = 'gs://{}/jars_parser.py'.format(args.bucket)
job_body['job']['pysparkJob']['args'][1] = args.bucket
- job_body['job']['pysparkJob']['args'][3] = (os.environ['project_name']).replace('_', '-')
+ job_body['job']['pysparkJob']['args'][3] = (os.environ['project_name']).lower().replace('_', '-')
job_body['job']['pysparkJob']['args'][5] = cluster_name
job_body['job']['pysparkJob']['args'][7] = cluster_version
job_body['job']['pysparkJob']['args'][9] = os.environ['conf_os_user']
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/project_terminate.py
index 66541e0..776fff7 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_terminate.py
@@ -33,7 +33,7 @@ def terminate_edge_node(project_name, service_base_name, region, zone):
try:
labels = [
{'sbn': service_base_name},
- {'project_tag': project_tag}
+ {'project_tag': project_name}
]
clusters_list = meta_lib.GCPMeta().get_dataproc_list(labels)
if clusters_list:
@@ -54,7 +54,7 @@ def terminate_edge_node(project_name, service_base_name, region, zone):
instances = GCPMeta().get_list_instances(zone, base)
if 'items' in instances:
for i in instances['items']:
- if 'project_tag' in i['labels'] and project_tag == i['labels']['project_tag']:
+ if 'project_tag' in i['labels'] and project_name == i['labels']['project_tag']:
GCPActions().remove_instance(i['name'], zone)
except Exception as err:
print('Error: {0}'.format(err))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org