You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2019/09/18 09:46:55 UTC
[incubator-dlab] branch DLAB-997 created (now 25aaa99)
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a change to branch DLAB-997
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 25aaa99 [DLAB-997] Added user tag to role creation on AWS
This branch includes the following new commits:
new 25aaa99 [DLAB-997] Added user tag to role creation on AWS
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-997] Added user tag to role creation
on AWS
Posted by dm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch DLAB-997
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 25aaa99f31a2cbae1cfd8135ca90399684f75795
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Wed Sep 18 12:46:42 2019 +0300
[DLAB-997] Added user tag to role creation on AWS
---
.../src/general/lib/aws/actions_lib.py | 72 ++++++++++++----------
.../scripts/aws/common_create_role_policy.py | 4 +-
.../src/general/scripts/aws/project_prepare.py | 10 +--
.../src/general/scripts/aws/ssn_prepare.py | 5 +-
4 files changed, 52 insertions(+), 39 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index c0db77b..249fea3 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -221,7 +221,8 @@ def create_rt(vpc_id, infra_tag_name, infra_tag_value, secondary):
ec2.create_route(DestinationCidrBlock='0.0.0.0/0', RouteTableId=rt_id, GatewayId=ig_id)
return rt_id
except Exception as err:
- logging.info("Unable to create Route Table: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
+ logging.info(
+ "Unable to create Route Table: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
append_result(str({"error": "Unable to create Route Table",
"error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
@@ -248,7 +249,8 @@ def create_security_group(security_group_name, vpc_id, security_group_rules, egr
try:
ec2 = boto3.resource('ec2')
tag_name = {"Key": "Name", "Value": security_group_name}
- group = ec2.create_security_group(GroupName=security_group_name, Description='security_group_name', VpcId=vpc_id)
+ group = ec2.create_security_group(GroupName=security_group_name, Description='security_group_name',
+ VpcId=vpc_id)
time.sleep(10)
create_tag(group.id, tag)
create_tag(group.id, tag_name)
@@ -266,7 +268,8 @@ def create_security_group(security_group_name, vpc_id, security_group_rules, egr
group.authorize_egress(IpPermissions=[rule])
return group.id
except Exception as err:
- logging.info("Unable to create security group: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
+ logging.info(
+ "Unable to create security group: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
append_result(str({"error": "Unable to create security group",
"error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
@@ -292,9 +295,9 @@ def create_route_by_id(subnet_id, vpc_id, peering_id, another_cidr):
routeExists = True
if not routeExists:
client.create_route(
- DestinationCidrBlock = another_cidr,
- VpcPeeringConnectionId = peering_id,
- RouteTableId = final_id)
+ DestinationCidrBlock=another_cidr,
+ VpcPeeringConnectionId=peering_id,
+ RouteTableId=final_id)
except Exception as err:
logging.info("Unable to create route: " + str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout))
append_result(str({"error": "Unable to create route",
@@ -313,9 +316,9 @@ def create_peer_routes(peering_id, service_base_name):
service_base_name)]}]).get('RouteTables')
for table in route_tables:
routes = table.get('Routes')
- routeExists=False
+ routeExists = False
for route in routes:
- if route.get('DestinationCidrBlock')==os.environ['conf_vpc2_cidr'].replace("'", ""):
+ if route.get('DestinationCidrBlock') == os.environ['conf_vpc2_cidr'].replace("'", ""):
routeExists = True
if not routeExists:
client.create_route(
@@ -324,9 +327,9 @@ def create_peer_routes(peering_id, service_base_name):
RouteTableId=table.get('RouteTableId'))
for table in route_tables2:
routes = table.get('Routes')
- routeExists=False
+ routeExists = False
for route in routes:
- if route.get('DestinationCidrBlock')==os.environ['conf_vpc_cidr'].replace("'", ""):
+ if route.get('DestinationCidrBlock') == os.environ['conf_vpc_cidr'].replace("'", ""):
routeExists = True
if not routeExists:
client.create_route(
@@ -523,7 +526,7 @@ def tag_emr_volume(cluster_id, node_name, billing_tag):
traceback.print_exc(file=sys.stdout)
-def create_iam_role(role_name, role_profile, region, service='ec2', tag=None):
+def create_iam_role(role_name, role_profile, region, service='ec2', tag=None, user_tag=None):
conn = boto3.client('iam')
try:
if region == 'cn-north-1':
@@ -540,6 +543,8 @@ def create_iam_role(role_name, role_profile, region, service='ec2', tag=None):
if tag:
conn.tag_role(RoleName=role_name, Tags=[tag])
conn.tag_role(RoleName=role_name, Tags=[{"Key": "Name", "Value": role_name}])
+ if user_tag:
+ conn.tag_role(RoleName=role_name, Tags=[user_tag])
if 'conf_billing_tag_key' in os.environ and 'conf_billing_tag_value' in os.environ:
conn.tag_role(RoleName=role_name, Tags=[{'Key': os.environ['conf_billing_tag_key'],
'Value': os.environ['conf_billing_tag_value']}])
@@ -866,7 +871,7 @@ def remove_detach_iam_policies(role_name, action=''):
logging.info("Unable to remove/detach IAM policy: " + str(err) + "\n Traceback: " + traceback.print_exc(
file=sys.stdout))
append_result(str({"error": "Unable to remove/detach IAM policy",
- "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
@@ -882,7 +887,7 @@ def remove_roles_and_profiles(role_name, role_profile_name):
logging.info("Unable to remove IAM role/profile: " + str(err) + "\n Traceback: " + traceback.print_exc(
file=sys.stdout))
append_result(str({"error": "Unable to remove IAM role/profile",
- "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
@@ -1082,7 +1087,7 @@ def remove_peering(tag_value):
try:
client = boto3.client('ec2')
tag_name = os.environ['conf_service_base_name'] + '-Tag'
- if os.environ['conf_duo_vpc_enable']=='true':
+ if os.environ['conf_duo_vpc_enable'] == 'true':
peering_id = client.describe_vpc_peering_connections(Filters=[
{'Name': 'tag-key', 'Values': [tag_name]},
{'Name': 'tag-value', 'Values': [tag_value]},
@@ -1102,6 +1107,7 @@ def remove_peering(tag_value):
"error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+
def remove_sgroups(tag_value):
try:
ec2 = boto3.resource('ec2')
@@ -1166,8 +1172,8 @@ def deregister_image(image_name='*'):
client = boto3.client('ec2')
for image in resource.images.filter(
Filters=[{'Name': 'name', 'Values': ['{}-*'.format(os.environ['conf_service_base_name'])]},
- {'Name': 'tag-value', 'Values': [os.environ['conf_service_base_name']]},
- {'Name': 'tag-value', 'Values': [image_name]}]):
+ {'Name': 'tag-value', 'Values': [os.environ['conf_service_base_name']]},
+ {'Name': 'tag-value', 'Values': [image_name]}]):
client.deregister_image(ImageId=image.id)
for device in image.block_device_mappings:
if device.get('Ebs'):
@@ -1288,7 +1294,7 @@ def remove_route_tables(tag_name, ssn=False):
logging.info("Unable to remove route table: " + str(err) + "\n Traceback: " + traceback.print_exc(
file=sys.stdout))
append_result(str({"error": "Unable to remove route table",
- "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
@@ -1302,7 +1308,7 @@ def remove_internet_gateways(vpc_id, tag_name, tag_value):
{'Name': 'tag-value', 'Values': [tag_value]}]).get('InternetGateways')
for i in response:
ig_id = i.get('InternetGatewayId')
- client.detach_internet_gateway(InternetGatewayId=ig_id,VpcId=vpc_id)
+ client.detach_internet_gateway(InternetGatewayId=ig_id, VpcId=vpc_id)
print("Internet gateway {0} has been detached from VPC {1}".format(ig_id, vpc_id.format))
client.delete_internet_gateway(InternetGatewayId=ig_id)
print("Internet gateway {} has been deleted successfully".format(ig_id))
@@ -1310,7 +1316,7 @@ def remove_internet_gateways(vpc_id, tag_name, tag_value):
logging.info("Unable to remove internet gateway: " + str(err) + "\n Traceback: " + traceback.print_exc(
file=sys.stdout))
append_result(str({"error": "Unable to remove internet gateway",
- "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
@@ -1325,7 +1331,7 @@ def remove_vpc_endpoints(vpc_id):
logging.info("Unable to remove VPC Endpoint: " + str(err) + "\n Traceback: " + traceback.print_exc(
file=sys.stdout))
append_result(str({"error": "Unable to remove VPC Endpoint",
- "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
@@ -1446,8 +1452,8 @@ def get_gitlab_cert(bucket, certfile):
def create_aws_config_files(generate_full_config=False):
try:
aws_user_dir = os.environ['AWS_DIR']
- logging.info(local("rm -rf " + aws_user_dir+" 2>&1", capture=True))
- logging.info(local("mkdir -p " + aws_user_dir+" 2>&1", capture=True))
+ logging.info(local("rm -rf " + aws_user_dir + " 2>&1", capture=True))
+ logging.info(local("mkdir -p " + aws_user_dir + " 2>&1", capture=True))
with open(aws_user_dir + '/config', 'w') as aws_file:
aws_file.write("[default]\n")
@@ -1459,8 +1465,8 @@ def create_aws_config_files(generate_full_config=False):
aws_file.write("aws_access_key_id = {}\n".format(os.environ['aws_access_key']))
aws_file.write("aws_secret_access_key = {}\n".format(os.environ['aws_secret_access_key']))
- logging.info(local("chmod 600 " + aws_user_dir + "/*"+" 2>&1", capture=True))
- logging.info(local("chmod 550 " + aws_user_dir+" 2>&1", capture=True))
+ logging.info(local("chmod 600 " + aws_user_dir + "/*" + " 2>&1", capture=True))
+ logging.info(local("chmod 550 " + aws_user_dir + " 2>&1", capture=True))
return True
except Exception as err:
@@ -1475,7 +1481,7 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
python_version = python_version[0:5]
if not os.path.exists('/opt/python/python' + python_version):
local('wget https://www.python.org/ftp/python/' + python_version +
- '/Python-' + python_version + '.tgz -O /tmp/Python-' + python_version + '.tgz' )
+ '/Python-' + python_version + '.tgz -O /tmp/Python-' + python_version + '.tgz')
local('tar zxvf /tmp/Python-' + python_version + '.tgz -C /tmp/')
with lcd('/tmp/Python-' + python_version):
local('./configure --prefix=/opt/python/python' + python_version +
@@ -1539,7 +1545,8 @@ def installing_python(region, bucket, user_name, cluster_name, application='', p
' install https://cntk.ai/PythonWheel/GPU/cntk-2.0rc3-cp{0}-cp{0}m-linux_x86_64.whl '
'--no-cache-dir'.format(python_without_dots[:2]))
local('sudo rm -rf /usr/bin/python{}-dp'.format(python_version[0:3]))
- local('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'.format(python_version, python_version[0:3]))
+ local('sudo ln -fs /opt/python/python{0}/bin/python{1} /usr/bin/python{1}-dp'.format(python_version,
+ python_version[0:3]))
def spark_defaults(args):
@@ -1550,8 +1557,9 @@ def spark_defaults(args):
local(""" sudo bash -c " sed -i '/^\s*$/d' """ + spark_def_path + """ " """)
local(""" sudo bash -c "sed -i '/spark.driver.extraClassPath/,/spark.driver.extraLibraryPath/s|"""
"""/usr|/opt/DATAENGINE-SERVICE_VERSION/jars/usr|g' """ + spark_def_path + """ " """)
- local(""" sudo bash -c "sed -i '/spark.yarn.dist.files/s/\/etc\/spark\/conf/\/opt\/DATAENGINE-SERVICE_VERSION\/CLUSTER\/conf/g' """
- + spark_def_path + """ " """)
+ local(
+ """ sudo bash -c "sed -i '/spark.yarn.dist.files/s/\/etc\/spark\/conf/\/opt\/DATAENGINE-SERVICE_VERSION\/CLUSTER\/conf/g' """
+ + spark_def_path + """ " """)
template_file = spark_def_path
with open(template_file, 'r') as f:
text = f.read()
@@ -1642,7 +1650,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
except Exception as err:
print('Error:', str(err))
sys.exit(1)
-
+
def configure_zeppelin_emr_interpreter(emr_version, cluster_name, region, spark_dir, os_user, yarn_dir, bucket,
user_name, endpoint_url, multiple_emrs):
@@ -1660,8 +1668,8 @@ def configure_zeppelin_emr_interpreter(emr_version, cluster_name, region, spark_
"/hadoop-aws*.jar /opt/" + \
"{0}/jars/usr/share/aws/aws-java-sdk/aws-java-sdk-s3-*.jar /opt/{0}" + \
"/jars/usr/lib/hadoop-lzo/lib/hadoop-lzo-*.jar".format(emr_version)
- #fix due to: Multiple py4j files found under ..../spark/python/lib
- #py4j-0.10.7-src.zip still in folder. Versions may varies.
+ # fix due to: Multiple py4j files found under ..../spark/python/lib
+ # py4j-0.10.7-src.zip still in folder. Versions may varies.
local('rm /opt/{0}/{1}/spark/python/lib/py4j-src.zip'.format(emr_version, cluster_name))
local('echo \"Configuring emr path for Zeppelin\"')
@@ -1779,7 +1787,7 @@ def configure_dataengine_spark(cluster_name, jars_dir, cluster_dir, datalake_ena
additional_spark_properties = local('diff --changed-group-format="%>" --unchanged-group-format="" '
'/tmp/{0}/notebook_spark-defaults_local.conf '
'{1}spark/conf/spark-defaults.conf | grep -v "^#"'.format(
- cluster_name, cluster_dir), capture=True)
+ cluster_name, cluster_dir), capture=True)
for property in additional_spark_properties.split('\n'):
local('echo "{0}" >> /tmp/{1}/notebook_spark-defaults_local.conf'.format(property, cluster_name))
local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf {1}spark/conf/spark-defaults.conf'.format(cluster_name,
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
index 15e8e87..1f914c1 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
@@ -35,6 +35,7 @@ parser.add_argument('--policy_file_name', type=str, default='')
parser.add_argument('--region', type=str, default='')
parser.add_argument('--infra_tag_name', type=str, default='')
parser.add_argument('--infra_tag_value', type=str, default='')
+parser.add_argument('--user_tag_value', type=str, default='')
args = parser.parse_args()
@@ -44,8 +45,9 @@ if __name__ == "__main__":
role_name = get_role_by_name(args.role_name)
if role_name == '':
tag = {"Key": args.infra_tag_name, "Value": args.infra_tag_value}
+ user_tag = {"Key": "user:tag", "Value": args.user_tag_value}
print("Creating role {0}, profile name {1}".format(args.role_name, args.role_profile_name))
- create_iam_role(args.role_name, args.role_profile_name, args.region, tag=tag)
+ create_iam_role(args.role_name, args.role_profile_name, args.region, tag=tag, user_tag=user_tag)
else:
print("ROLE AND ROLE PROFILE ARE ALREADY CREATED")
print("ROLE {} created. IAM group {} created".format(args.role_name, args.role_profile_name))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index c09cea2..b4aeb43 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -175,11 +175,12 @@ if __name__ == "__main__":
try:
logging.info('[CREATE EDGE ROLES]')
print('[CREATE EDGE ROLES]')
+ user_tag = "{0}:{0}-{1}-edge-Role".format(project_conf['service_base_name'], project_conf['project_name'])
params = "--role_name {} --role_profile_name {} --policy_name {} --region {} --infra_tag_name {} " \
- "--infra_tag_value {}" \
+ "--infra_tag_value {} --user_tag_value {}" \
.format(project_conf['edge_role_name'], project_conf['edge_role_profile_name'],
project_conf['edge_policy_name'], os.environ['aws_region'], project_conf['tag_name'],
- project_conf['service_base_name'])
+ project_conf['service_base_name'], user_tag)
try:
local("~/scripts/{}.py {}".format('common_create_role_policy', params))
except:
@@ -193,11 +194,12 @@ if __name__ == "__main__":
try:
logging.info('[CREATE BACKEND (NOTEBOOK) ROLES]')
print('[CREATE BACKEND (NOTEBOOK) ROLES]')
+ user_tag = "{0}:{0}-{1}-nb-de-Role".format(project_conf['service_base_name'], project_conf['project_name'])
params = "--role_name {} --role_profile_name {} --policy_name {} --region {} --infra_tag_name {} " \
- "--infra_tag_value {}" \
+ "--infra_tag_value {} --user_tag_value {}" \
.format(project_conf['notebook_dataengine_role_name'], project_conf['notebook_dataengine_role_profile_name'],
project_conf['notebook_dataengine_policy_name'], os.environ['aws_region'], project_conf['tag_name'],
- project_conf['service_base_name'])
+ project_conf['service_base_name'], user_tag)
try:
local("~/scripts/{}.py {}".format('common_create_role_policy', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index 1868fd1..061100a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@ -67,6 +67,7 @@ if __name__ == "__main__":
shared_bucket_name = shared_bucket_name_tag.lower().replace('_', '-')
tag_name = service_base_name + '-Tag'
tag2_name = service_base_name + '-secondary-Tag'
+ user_tag = "{}:{}-ssn-Role"
instance_name = service_base_name + '-ssn'
region = os.environ['aws_region']
zone_full = os.environ['aws_region'] + os.environ['aws_zone']
@@ -291,9 +292,9 @@ if __name__ == "__main__":
logging.info('[CREATE ROLES]')
print('[CREATE ROLES]')
params = "--role_name {} --role_profile_name {} --policy_name {} --policy_file_name {} --region {} " \
- "--infra_tag_name {} --infra_tag_value {}".\
+ "--infra_tag_name {} --infra_tag_value {} --user_tag_value {}".\
format(role_name, role_profile_name, policy_name, policy_path, os.environ['aws_region'], tag_name,
- service_base_name)
+ service_base_name, user_tag)
try:
local("~/scripts/{}.py {}".format('common_create_role_policy', params))
except:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org