You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/07/21 11:47:14 UTC
[incubator-datalab] 01/01: [DATALAB-2409] - DataLab deployment
script refactored
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 66eb986f6220430c89829ec6349df35b1fc3046c
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Wed Jul 21 14:46:49 2021 +0300
[DATALAB-2409] - DataLab deployment script refactored
---
.../scripts/deploy_datalab.py | 399 +++++++++++++--------
.../src/base/scripts/install_prerequisites.py | 13 -
.../src/general/conf/datalab.ini | 4 -
.../src/general/files/aws/base_Dockerfile | 3 +-
.../src/general/files/azure/base_Dockerfile | 3 +-
.../src/general/files/gcp/base_Dockerfile | 3 +-
.../src/general/files/os/debian/sources.list | 56 ---
.../src/general/files/os/redhat/sources.list | 83 -----
.../src/general/lib/os/debian/common_lib.py | 7 +-
.../src/general/lib/os/redhat/common_lib.py | 7 -
...common_notebook_configure_dataengine-service.py | 4 +-
.../src/general/scripts/aws/ssn_prepare.py | 15 +-
...common_notebook_configure_dataengine-service.py | 5 +-
infrastructure-provisioning/src/ssn/fabfile.py | 8 +-
.../src/ssn/scripts/configure_docker.py | 7 -
15 files changed, 256 insertions(+), 361 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index e3bdd4c..cf322a6 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -20,162 +20,236 @@
#
# ******************************************************************************
+"""Examples How to deploy DataLab for different cloud providers.
+
+``GCP`` example::
+
+ $ infrastructure-provisioning/scripts/deploy_datalab.py \
+ --conf_service_base_name <SERVICE_NAME> \
+ --conf_os_family debian \
+ --action create \
+ --key_path /home/ubuntu/.ssh \
+ --conf_key_name gcp \
+ --billing_dataset_name billing \
+ gcp \
+ --gcp_ssn_instance_size n1-standard-2 \
+ --gcp_project_id <PROJECT_ID>\
+ --gcp_service_account_path /home/ubuntu/secret.json\
+ --gcp_region us-west1\
+ --gcp_zone us-west1-a
+
+``AWS`` example::
+
+ $ infrastructure-provisioning/scripts/deploy_datalab.py\
+ --conf_service_base_name datalab-test\
+ --conf_os_family debian\
+ --action create \
+ --key_path /path/to/key/\
+ --conf_key_name key_name\
+ --conf_tag_resource_id datalab\
+ aws\
+ --aws_vpc_id vpc-xxxxx\
+ --aws_subnet_id subnet-xxxxx\
+ --aws_security_groups_ids sg-xxxxx,sg-xxxx\
+ --aws_access_key XXXXXXX\
+ --aws_secret_access_key XXXXXXXXXX\
+ --aws_region xx-xxxxx-x\
+ --aws_account_id xxxxxxxx\
+ --aws_billing_bucket billing_bucket\
+ --aws_report_path /billing/directory/\
+
+"""
import argparse
import os
import subprocess
-from fabric import *
-from invoke import task
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--conf_service_base_name', type=str, help='unique name for DataLab environment')
-parser.add_argument('--conf_network_type', type=str, default='',
- help='Define in which network DataLab will be deployed. '
- 'Possible options: public|private')
-parser.add_argument('--conf_vpc_cidr', type=str, default='', help='CIDR of VPC')
-parser.add_argument('--conf_vpc2_cidr', type=str, default='', help='CIDR of secondary VPC')
-parser.add_argument('--conf_allowed_ip_cidr', type=str, default='', help='Comma-separated CIDR of IPs which will have '
- 'access to SSN')
-parser.add_argument('--conf_user_subnets_range', type=str, default='', help='Range of subnets which will be using for '
- 'users environments. For example: '
- '10.10.0.0/24 - 10.10.10.0/24')
-parser.add_argument('--conf_private_subnet_prefix', type=str, default='24', help='Private subnet prefix')
-parser.add_argument('--conf_additional_tags', type=str, default='', help='Additional tags in format '
- '"Key1:Value1;Key2:Value2"')
-parser.add_argument('--conf_image_enabled', type=str, default='', help='Enable or Disable creating image at first time')
-parser.add_argument('--aws_user_predefined_s3_policies', type=str, default='', help='Predefined policies for users '
- 'instances')
-parser.add_argument('--aws_access_key', type=str, default='', help='AWS Access Key ID')
-parser.add_argument('--aws_secret_access_key', type=str, default='', help='AWS Secret Access Key')
-parser.add_argument('--aws_region', type=str, default='', help='AWS region')
-parser.add_argument('--aws_zone', type=str, default='', help='AWS zone')
-parser.add_argument('--azure_region', type=str, default='', help='Azure region')
-parser.add_argument('--gcp_region', type=str, default='', help='GCP region')
-parser.add_argument('--gcp_zone', type=str, default='', help='GCP zone')
-parser.add_argument('--conf_os_family', type=str, default='',
- help='Operating system type. Available options: debian, redhat')
-parser.add_argument('--conf_cloud_provider', type=str, default='',
- help='Where DataLab should be deployed. Available options: aws, azure, gcp')
-parser.add_argument('--ssn_hosted_zone_name', type=str, default='', help='Name of hosted zone')
-parser.add_argument('--ssn_hosted_zone_id', type=str, default='', help='ID of hosted zone')
-parser.add_argument('--ssn_subdomain', type=str, default='', help='Subdomain name')
-parser.add_argument('--ssn_assume_role_arn', type=str, default='', help='Role ARN for creating Route53 record in '
- 'different AWS account')
-parser.add_argument('--ssl_cert_path', type=str, default='', help='Full path to SSL certificate')
-parser.add_argument('--ssl_key_path', type=str, default='', help='Full path to key for SSL certificate')
-parser.add_argument('--aws_vpc_id', type=str, default='', help='AWS VPC ID')
-parser.add_argument('--conf_duo_vpc_enable', type=str, default='false', help='Duo VPC scheme enable(true|false)')
-parser.add_argument('--aws_vpc2_id', type=str, default='', help='Secondary AWS VPC ID')
-parser.add_argument('--aws_peering_id', type=str, default='', help='Amazon peering connection id')
-parser.add_argument('--azure_vpc_name', type=str, default='', help='Azure VPC Name')
-parser.add_argument('--gcp_vpc_name', type=str, default='', help='GCP VPC Name')
-parser.add_argument('--aws_subnet_id', type=str, default='', help='AWS Subnet ID')
-parser.add_argument('--azure_subnet_name', type=str, default='', help='Azure Subnet Name')
-parser.add_argument('--gcp_subnet_name', type=str, default='', help='GCP Subnet Name')
-parser.add_argument('--aws_security_groups_ids', type=str, default='', help='One of more comma-separated Security '
- 'groups IDs for SSN')
-parser.add_argument('--azure_security_group_name', type=str, default='', help='One Security'
- 'group name for SSN')
-parser.add_argument('--azure_edge_security_group_name', type=str, default='', help='One Security '
- 'group name for Edge node')
-parser.add_argument('--gcp_firewall_name', type=str, default='', help='One of more comma-separated GCP Firewall rules '
- 'for SSN')
-parser.add_argument('--key_path', type=str, default='', help='Path to admin key (WITHOUT KEY NAME)')
-parser.add_argument('--conf_key_name', type=str, default='', help='Admin key name (WITHOUT ".pem")')
-parser.add_argument('--workspace_path', type=str, default='', help='Admin key name (WITHOUT ".pem")')
-parser.add_argument('--conf_tag_resource_id', type=str, default='datalab', help='The name of user tag')
-parser.add_argument('--conf_billing_tag', type=str, default='datalab', help='Billing tag')
-parser.add_argument('--aws_ssn_instance_size', type=str, default='t2.large', help='The SSN instance shape')
-parser.add_argument('--azure_ssn_instance_size', type=str, default='Standard_DS2_v2', help='The SSN instance shape')
-parser.add_argument('--gcp_ssn_instance_size', type=str, default='n1-standard-2', help='The SSN instance shape')
-parser.add_argument('--aws_account_id', type=str, default='', help='The ID of Amazon account')
-parser.add_argument('--aws_billing_bucket', type=str, default='', help='The name of S3 bucket where billing reports '
- 'will be placed.')
-parser.add_argument('--aws_job_enabled', type=str, default='false', help='Billing format. Available options: '
- 'true (aws), false(epam)')
-parser.add_argument('--aws_report_path', type=str, default='', help='The path to billing reports directory in S3 '
- 'bucket')
-parser.add_argument('--azure_resource_group_name', type=str, default='', help='Name of Resource group in Azure')
-parser.add_argument('--azure_auth_path', type=str, default='', help='Full path to Azure credentials JSON file')
-parser.add_argument('--azure_datalake_enable', type=str, default='', help='Provision DataLake storage account')
-parser.add_argument('--azure_ad_group_id', type=str, default='', help='ID of Azure AD group')
-parser.add_argument('--azure_offer_number', type=str, default='', help='Azure offer number')
-parser.add_argument('--azure_currency', type=str, default='', help='Azure currency code')
-parser.add_argument('--azure_locale', type=str, default='', help='Azure locale')
-parser.add_argument('--azure_application_id', type=str, default='', help='Azure login application ID')
-parser.add_argument('--azure_validate_permission_scope', type=str, default='true', help='Azure permission scope '
- 'validation(true|false).')
-parser.add_argument('--azure_oauth2_enabled', type=str, default='false', help='Using OAuth2 for logging in DataLab')
-parser.add_argument('--azure_region_info', type=str, default='', help='Azure region info')
-parser.add_argument('--azure_source_vpc_name', type=str, default='', help='Azure VPC source Name')
-parser.add_argument('--azure_source_resource_group_name', type=str, default='', help='Azure source resource group')
-parser.add_argument('--gcp_project_id', type=str, default='', help='The project ID in Google Cloud Platform')
-parser.add_argument('--gcp_service_account_path', type=str, default='', help='The project ID in Google Cloud Platform')
-parser.add_argument('--datalab_id', type=str, default="'resource_tags_user_user_tag'",
- help='Column name in report file that contains '
- 'datalab id tag')
-parser.add_argument('--usage_date', type=str, default='line_item_usage_start_date',
- help='Column name in report file that contains '
- 'usage date tag')
-parser.add_argument('--product', type=str, default='product_product_name',
- help='Column name in report file that contains '
- 'product name tag')
-parser.add_argument('--usage_type', type=str, default='line_item_usage_type',
- help='Column name in report file that contains '
- 'usage type tag')
-parser.add_argument('--usage', type=str, default='line_item_usage_amount',
- help='Column name in report file that contains '
- 'usage tag')
-parser.add_argument('--cost', type=str, default='line_item_blended_cost',
- help='Column name in report file that contains cost tag')
-parser.add_argument('--resource_id', type=str, default='line_item_resource_id',
- help='Column name in report file that contains '
- 'datalab resource id tag')
-parser.add_argument('--ldap_hostname', type=str, default='localhost', help='Ldap instance hostname')
-parser.add_argument('--ldap_dn', type=str, default='dc=example,dc=com',
- help='Ldap distinguished name')
-parser.add_argument('--ldap_ou', type=str, default='ou=People', help='Ldap organisation unit')
-parser.add_argument('--ldap_service_username', type=str, default='cn=service-user', help='Ldap service user name')
-parser.add_argument('--ldap_service_password', type=str, default='service-user-password',
- help='Ldap password for admin user')
-parser.add_argument('--keycloak_realm_name', type=str, default='datalab', help='Keycloak Realm name')
-parser.add_argument('--keycloak_auth_server_url', type=str, default='datalab', help='Keycloak auth server URL')
-parser.add_argument('--keycloak_client_name', type=str, default='datalab', help='Keycloak client name')
-parser.add_argument('--keycloak_client_secret', type=str, default='datalab', help='Keycloak client secret')
-parser.add_argument('--keycloak_user', type=str, default='datalab', help='Keycloak user')
-parser.add_argument('--keycloak_user_password', type=str, default='keycloak-user-password',
- help='Keycloak user password')
-parser.add_argument('--tags', type=str, default='line_item_operation,line_item_line_item_description',
- help='Column name in report file that '
- 'contains tags')
-parser.add_argument('--billing_dataset_name', type=str, default='', help='Name of GCP dataset (BigQuery service)'
- ' for billing')
-parser.add_argument('--default_endpoint_name', type=str, default='local', help='Name of localhost provisioning service,'
- 'that created by default')
-parser.add_argument('--conf_stepcerts_enabled', type=str, default='false', help='Enable or disable step certificates')
-parser.add_argument('--conf_stepcerts_root_ca', type=str, default='', help='Step root CA')
-parser.add_argument('--conf_stepcerts_kid', type=str, default='', help='Step KID')
-parser.add_argument('--conf_stepcerts_kid_password', type=str, default='', help='Step KID password')
-parser.add_argument('--conf_stepcerts_ca_url', type=str, default='', help='Step CA URL')
-parser.add_argument('--conf_letsencrypt_enabled', type=str, default='false',
- help='Enable or disable Let`s Encrypt certificates')
-parser.add_argument('--conf_repository_user', type=str, default='',
- help='user to access repository (used for jars download)')
-parser.add_argument('--conf_release_tag', type=str, default='2.5',
- help='tag used for jars download')
-parser.add_argument('--conf_repository_pass', type=str, default='',
- help='password to access repository (used for jars download)')
-parser.add_argument('--conf_repository_address', type=str, default='',
- help='address to access repository (used for jars download)')
-parser.add_argument('--conf_letsencrypt_domain_name', type=str, default='', help='Domain names to apply. '
- 'For multiple domains enter a comma separated list of domains as a parameter'
- 'ssn.domain_name will be used for ssn_node, DNS A record have to exist during deployment')
-parser.add_argument('--conf_letsencrypt_email', type=str, default='', help='Email that will be entered during '
- 'certificate obtaining and can be user for urgent renewal and security notices. '
- 'Use comma to register multiple emails, e.g. u1@example.com,u2@example.com.')
-parser.add_argument('--action', required=True, type=str, default='', choices=['build', 'deploy', 'create', 'terminate'],
- help='Available options: build, deploy, create, terminate')
-args = parser.parse_args()
+import sys
+
+BOOL_CHOICES_LIST = ['true', 'false']
+OS_DISTRO_LIST = ['debian', 'redhat']
+NETWORK_TYPE_LIST = ['public', 'private']
+
+
+def build_parser():
+ parser = argparse.ArgumentParser(description='DataLab Self-Service Node deployment',
+ prog='deploy_datalab')
+ # optional arguments
+ parser.add_argument('--conf_network_type', type=str, default='public',
+ help='''Type of network. Define in which network DataLab will be deployed.
+ (valid choices: %s)''' % NETWORK_TYPE_LIST,
+ choices=NETWORK_TYPE_LIST)
+ parser.add_argument('--conf_vpc_cidr', type=str, default='172.31.0.0/16', help='CIDR of VPC')
+ parser.add_argument('--conf_vpc2_cidr', type=str, help='CIDR of secondary VPC')
+ parser.add_argument('--conf_allowed_ip_cidr', type=str, default='0.0.0.0/0',
+ help='Comma-separated CIDR of IPs which will have access to SSN')
+ parser.add_argument('--conf_user_subnets_range', type=str,
+ help='''Range of subnets which will be using for users environments.
+ For example: 10.10.0.0/24 - 10.10.10.0/24''')
+ parser.add_argument('--conf_private_subnet_prefix', type=str, default='24', help='Private subnet prefix')
+ parser.add_argument('--conf_additional_tags', type=str,
+ help='Additional tags in format "Key1:Value1;Key2:Value2"')
+ parser.add_argument('--conf_image_enabled', type=str,
+ help='Enable or Disable creating image at first time')
+ parser.add_argument('--conf_os_family', type=str, default='debian', choices=OS_DISTRO_LIST,
+ help='Operating system distribution. (valid choices: %s)' % OS_DISTRO_LIST)
+ parser.add_argument('--ssn_hosted_zone_name', type=str, help='Name of hosted zone')
+ parser.add_argument('--ssn_hosted_zone_id', type=str, help='ID of hosted zone')
+ parser.add_argument('--ssn_subdomain', type=str, help='Subdomain name')
+ parser.add_argument('--ssl_cert_path', type=str, help='Full path to SSL certificate')
+ parser.add_argument('--ssl_key_path', type=str, help='Full path to key for SSL certificate')
+ parser.add_argument('--workspace_path', type=str, default='', help='Docker workspace path')
+ parser.add_argument('--conf_tag_resource_id', type=str, default='datalab', help='The name of user tag')
+ parser.add_argument('--conf_billing_tag', type=str, default='datalab', help='Billing tag')
+ parser.add_argument('--datalab_id', type=str, default='resource_tags_user_user_tag',
+ help='Column name in report file that contains datalab id tag')
+ parser.add_argument('--usage_date', type=str, default='line_item_usage_start_date',
+ help='Column name in report file that contains usage date tag')
+ parser.add_argument('--product', type=str, default='product_product_name',
+ help='Column name in report file that contains product name tag')
+ parser.add_argument('--usage_type', type=str, default='line_item_usage_type',
+ help='Column name in report file that contains usage type tag')
+ parser.add_argument('--usage', type=str, default='line_item_usage_amount',
+ help='Column name in report file that contains usage tag')
+ parser.add_argument('--cost', type=str, default='line_item_blended_cost',
+ help='Column name in report file that contains cost tag')
+ parser.add_argument('--resource_id', type=str, default='line_item_resource_id',
+ help='Column name in report file that contains datalab resource id tag')
+
+ parser.add_argument('--tags', type=str, default='line_item_operation,line_item_line_item_description',
+ help='Column name in report file that contains tags')
+ parser.add_argument('--conf_stepcerts_enabled', type=str, default='false',
+ help='Enable or disable step certificates. (valid choices: %s)' % BOOL_CHOICES_LIST,
+ choices=BOOL_CHOICES_LIST)
+ parser.add_argument('--conf_stepcerts_root_ca', type=str, help='Step root CA')
+ parser.add_argument('--conf_stepcerts_kid', type=str, help='Step KID')
+ parser.add_argument('--conf_stepcerts_kid_password', type=str, help='Step KID password')
+ parser.add_argument('--conf_stepcerts_ca_url', type=str, help='Step CA URL')
+ parser.add_argument('--conf_letsencrypt_enabled', type=str, default='false',
+ help='Enable or disable Let`s Encrypt certificates. (valid choices: %s)' % BOOL_CHOICES_LIST,
+ choices=BOOL_CHOICES_LIST)
+ parser.add_argument('--conf_letsencrypt_domain_name', type=str,
+ help='''Domain names to apply. For multiple domains enter a comma separated list of domains
+ as a parameter. ssn.domain_name will be used for ssn_node,DNS A record have to exist during deployment''')
+ parser.add_argument('--conf_letsencrypt_email', type=str, help='''Email that will be entered during
+ certificate obtaining and can be user for urgent renewal and security notices. Use comma to register
+ multiple emails, e.g. u1@example.com,u2@example.com.''')
+ parser.add_argument('--conf_repository_user', type=str, default='',
+ help='user to access repository (used for jars download)')
+ parser.add_argument('--conf_release_tag', type=str, default='2.5',
+ help='tag used for jars download')
+ parser.add_argument('--conf_repository_pass', type=str, default='',
+ help='password to access repository (used for jars download)')
+ parser.add_argument('--conf_repository_address', type=str, default='',
+ help='address to access repository (used for jars download)')
+
+ required_args = parser.add_argument_group('Required arguments')
+ required_args.add_argument('--conf_service_base_name', type=str,
+ help='Unique name for DataLab environment', required=True)
+ required_args.add_argument('--action', type=str, help='Action to perform',
+ choices=['build', 'deploy', 'create', 'terminate'], required=True)
+ required_args.add_argument('--key_path', type=str, help='Path to admin key (WITHOUT KEY NAME)', required=True)
+ required_args.add_argument('--conf_key_name', type=str, help='Admin key name (WITHOUT ".pem")', required=True)
+ required_args.add_argument('--keycloak_auth_server_url', type=str, default='datalab',
+ help='Keycloak auth server URL', required=True)
+ required_args.add_argument('--keycloak_realm_name', type=str, help='Keycloak Realm name', required=True)
+ required_args.add_argument('--keycloak_client_name', type=str, default='datalab',
+ help='Keycloak client name', required=True)
+ required_args.add_argument('--keycloak_client_secret', type=str, default='datalab',
+ help='Keycloak client secret', required=True)
+ required_args.add_argument('--keycloak_user', type=str, default='datalab', help='Keycloak user', required=True)
+ required_args.add_argument('--keycloak_user_password', type=str, default='keycloak-user-password',
+ help='Keycloak user password', required=True)
+ required_args.add_argument('--default_endpoint_name', type=str, default='local',
+ help='Name of localhost provisioning service, that created by default', required=True)
+
+ # subparsers
+ subparsers = parser.add_subparsers(dest='conf_cloud_provider', required=True, help='sub-command help',
+ description='''These are the subcommands for deploying resources
+ in a specific cloud provider''')
+
+ # --------- aws subcommand ----------------------
+ aws_parser = subparsers.add_parser('aws')
+ aws_parser.add_argument('--aws_user_predefined_s3_policies', type=str,
+ help='Predefined policies for users instances')
+ aws_parser.add_argument('--aws_access_key', type=str,
+ help='''AWS Access Key ID. reuqired in case of deployment with IAM user DataLab
+ deployment script is executed on local machine and uses
+ IAM user permissions to create resources in AWS.''')
+ aws_parser.add_argument('--aws_secret_access_key', type=str, help='AWS Secret Access Key')
+ aws_parser.add_argument('--ssn_assume_role_arn', type=str,
+ help='Role ARN for creating Route53 record in different AWS account')
+ aws_parser.add_argument('--aws_vpc_id', type=str, help='AWS VPC ID')
+ aws_parser.add_argument('--conf_duo_vpc_enable', type=str, default='false',
+ help='Duo VPC scheme enable. (valid choices: %s)' % BOOL_CHOICES_LIST,
+ choices=BOOL_CHOICES_LIST)
+ aws_parser.add_argument('--aws_vpc2_id', type=str, help='Secondary AWS VPC ID')
+ aws_parser.add_argument('--aws_peering_id', type=str, help='Amazon peering connection id')
+ aws_parser.add_argument('--aws_subnet_id', type=str, help='AWS Subnet ID')
+ aws_parser.add_argument('--aws_security_groups_ids', type=str,
+ help='One of more comma-separated Security groups IDs for SSN')
+ aws_parser.add_argument('--aws_billing_bucket', type=str,
+ help='The name of S3 bucket where billing reports will be placed.')
+ aws_parser.add_argument('--aws_job_enabled', type=str, default='false', choices=BOOL_CHOICES_LIST,
+ help='Billing format. (valid choices: %s)' % BOOL_CHOICES_LIST)
+ aws_parser.add_argument('--aws_report_path', type=str, help='The path to billing reports directory in S3 bucket')
+
+ aws_required_args = aws_parser.add_argument_group('Required arguments')
+ aws_required_args.add_argument('--aws_region', type=str, required=True, help='AWS region')
+ aws_required_args.add_argument('--aws_zone', type=str, required=True, help='AWS zone')
+ aws_required_args.add_argument('--aws_ssn_instance_size', type=str, required=True, default='t2.large',
+ help='The SSN instance shape')
+ aws_required_args.add_argument('--aws_account_id', type=str, required=True, help='The ID of Amazon account')
+
+ # --------azure subcommand -------------------------
+ azure_parser = subparsers.add_parser('azure')
+ azure_parser.add_argument('--azure_vpc_name', type=str, help='Azure VPC Name')
+ azure_parser.add_argument('--azure_subnet_name', type=str, help='Azure Subnet Name')
+ azure_parser.add_argument('--azure_security_group_name', type=str, help='One Security group name for SSN')
+ azure_parser.add_argument('--azure_edge_security_group_name', type=str,
+ help='One Security group name for Edge node')
+ azure_parser.add_argument('--azure_resource_group_name', type=str, help='Name of Resource group in Azure')
+ azure_parser.add_argument('--azure_datalake_enable', type=str, default='false', choices=BOOL_CHOICES_LIST,
+ help='Provision DataLake storage account. (valid choices: %s)' % BOOL_CHOICES_LIST)
+ azure_parser.add_argument('--azure_ad_group_id', type=str, help='ID of Azure AD group')
+ azure_parser.add_argument('--azure_offer_number', type=str, help='Azure offer number')
+ azure_parser.add_argument('--azure_currency', type=str, help='Azure currency code')
+ azure_parser.add_argument('--azure_locale', type=str, help='Azure locale')
+ azure_parser.add_argument('--azure_application_id', type=str, help='Azure login application ID')
+ azure_parser.add_argument('--azure_validate_permission_scope', type=str, default='true',
+ choices=BOOL_CHOICES_LIST,
+ help='Azure permission scope validation. (valid choices: %s)' % BOOL_CHOICES_LIST)
+ azure_parser.add_argument('--azure_oauth2_enabled', type=str, default='false', choices=BOOL_CHOICES_LIST,
+ help='Using OAuth2 for logging in DataLab. (valid choices: %s)' % BOOL_CHOICES_LIST)
+ azure_parser.add_argument('--azure_region_info', type=str, help='Azure region info')
+ azure_parser.add_argument('--azure_source_vpc_name', type=str, help='Azure VPC source Name')
+ azure_parser.add_argument('--azure_source_resource_group_name', type=str, help='Azure source resource group')
+
+ azure_required_args = azure_parser.add_argument_group('Required arguments')
+ azure_required_args.add_argument('--azure_region', type=str, required=True, help='Azure region')
+ azure_required_args.add_argument('--azure_ssn_instance_size', type=str, default='Standard_DS2_v2', required=True,
+ help='The SSN instance shape')
+ azure_required_args.add_argument('--azure_auth_path', type=str, required=True,
+ help='Full path to Azure credentials JSON file')
+
+ # --------gcp subcommand -----------------------------
+ gcp_parser = subparsers.add_parser('gcp')
+ gcp_parser.add_argument('--billing_dataset_name', type=str,
+ help='Name of GCP dataset (BigQuery service) for billing')
+ gcp_parser.add_argument('--gcp_subnet_name', type=str, help='GCP Subnet Name')
+ gcp_parser.add_argument('--gcp_vpc_name', type=str, help='GCP VPC Name')
+ gcp_parser.add_argument('--gcp_firewall_name', type=str,
+ help='One of more comma-separated GCP Firewall rules for SSN')
+
+ gcp_required_args = gcp_parser.add_argument_group('Required arguments')
+ gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region')
+ gcp_required_args.add_argument('--gcp_zone', type=str, required=True, help='GCP zone')
+ gcp_required_args.add_argument('--gcp_ssn_instance_size', type=str, required=True, default='n1-standard-2',
+ help='The SSN instance shape')
+ gcp_required_args.add_argument('--gcp_project_id', type=str, required=True,
+ help='The project ID in Google Cloud Platform')
+ gcp_required_args.add_argument('--gcp_service_account_path', type=str, required=True,
+ help='The project ID in Google Cloud Platform')
+ return parser
def generate_docker_command():
@@ -207,11 +281,15 @@ def generate_docker_command():
def build_docker_images(args):
# Building base and ssn docker images
- subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
- 'infrastructure-provisioning/src/general/files/{1}/'
- 'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True)
- subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
- 'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True)
+ subprocess.run(
+ 'cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
+ 'infrastructure-provisioning/src/general/files/{1}/'
+ 'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider,
+ args.workspace_path), shell=True, check=True)
+ subprocess.run(
+ 'cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
+ 'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider,
+ args.workspace_path), shell=True, check=True)
def deploy_datalab(args):
@@ -227,6 +305,13 @@ def terminate_datalab(args):
if __name__ == "__main__":
+ parser = build_parser()
+ args = parser.parse_args()
+
+ if args.aws_secret_access_key or args.aws_access_key:
+ if not (args.aws_secret_access_key and args.aws_access_key):
+ sys.exit('Please provide both arguments: --aws_secret_access_key and --aws_access_key')
+
if not args.workspace_path:
print("Workspace path isn't set, using current directory: {}".format(os.environ['PWD']))
args.workspace_path = os.environ['PWD']
diff --git a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
index 7b747b2..b75ae24 100644
--- a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
+++ b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
@@ -43,25 +43,12 @@ parser.add_argument('--region', type=str, default='')
args = parser.parse_args()
-def create_china_pip_conf_file(conn):
- if not exists(conn,'/home/{}/pip_china_ensured'.format(args.user)):
- conn.sudo('touch /etc/pip.conf')
- conn.sudo('echo "[global]" >> /etc/pip.conf')
- conn.sudo('echo "timeout = 600" >> /etc/pip.conf')
- conn.sudo('echo "index-url = https://{}/simple/" >> /etc/pip.conf'.format(os.environ['conf_pypi_mirror']))
- conn.sudo('echo "trusted-host = {}" >> /etc/pip.conf'.format(os.environ['conf_pypi_mirror']))
- conn.sudo('touch /home/{}/pip_china_ensured'.format(args.user))
-
if __name__ == "__main__":
print("Configure connections")
global conn
conn = init_datalab_connection(args.hostname, args.user, args.keyfile)
deeper_config = json.loads(args.additional_config)
- if args.region == 'cn-north-1':
- change_pkg_repos()
- create_china_pip_conf_file()
-
print("Updating hosts file")
update_hosts_file(args.user)
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index f9d2d1b..4881a03 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -40,8 +40,6 @@ key_dir = /root/keys/
lifecycle_stage = dev
### The name of user for tag, which will be set for all resources
# tag_resource_id = user:tag
-### Pypi mirror for China
-pypi_mirror = pypi.doubanio.com
### Name of own GitLab SSL certificate
gitlab_certfile = datalab-gitlab.crt
### Enable or Disable creating image at first time
@@ -262,8 +260,6 @@ scala_version = 2.12.8
livy_version = 0.3.0
### If it is true, Livy will be used on Zeppelin notebook
multiple_clusters = false
-### R China mirror
-r_mirror = http://mirror.lzu.edu.cn/CRAN/
### NVidia driver version for Tensor/DeepLearning notebooks
nvidia_version = 418.126.02
### Caffe library version for DeepLearning notebook
diff --git a/infrastructure-provisioning/src/general/files/aws/base_Dockerfile b/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
index 9cad9c4..70870b8 100644
--- a/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/aws/base_Dockerfile
@@ -41,7 +41,7 @@ ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
# Install any python dependencies
-RUN python3 -m pip install -UI qtconsole==4.7.7 pip==21.0.1 && \
+RUN python3 -m pip install -UI qtconsole==5.1.1 pip==21.1.2 && \
python3 -m pip install boto3 backoff patchwork fabric fabvenv awscli argparse requests ujson jupyter pycryptodome
# Configuring ssh for user
@@ -79,7 +79,6 @@ COPY ${SRC_PATH}general/scripts/aws/common_* /root/scripts/
COPY ${SRC_PATH}general/lib/aws/* /usr/lib/python3.8/datalab/
COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
-COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
COPY ${SRC_PATH}project/templates/locations/ /root/locations/
diff --git a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
index 3d608de..2070c05 100644
--- a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
@@ -30,7 +30,7 @@ RUN apt-get update && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install any python dependencies
-RUN python3 -m pip install -UI qtconsole==4.7.7 pip==21.0.1 && \
+RUN python3 -m pip install -UI qtconsole==5.1.1 pip==21.1.2 && \
python3 -m pip install backoff patchwork fabric fabvenv argparse requests ujson jupyter pycryptodome azure==2.0.0 azure-mgmt-authorization pyyaml
# Configuring ssh for user
@@ -68,7 +68,6 @@ COPY ${SRC_PATH}general/scripts/azure/common_* /root/scripts/
COPY ${SRC_PATH}general/lib/azure/* /usr/lib/python3.8/datalab/
COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
-COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
COPY ${SRC_PATH}project/templates/locations/ /root/locations/
diff --git a/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
index 2059bce..bb4a81c 100644
--- a/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/base_Dockerfile
@@ -41,7 +41,7 @@ ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
# Install any python dependencies
-RUN python3 -m pip install -UI pip==21.0.1 && \
+RUN python3 -m pip install -UI pip==21.1.2 && \
python3 -m pip install -U six patchwork configparser boto3 backoff fabric fabvenv argparse ujson jupyter pycryptodome google-api-python-client google-cloud-storage \
pyyaml google-auth-httplib2 oauth2client
@@ -79,7 +79,6 @@ COPY ${SRC_PATH}general/scripts/gcp/common_* /root/scripts/
COPY ${SRC_PATH}general/lib/gcp/* /usr/lib/python3.8/datalab/
COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python3.8/datalab/common_lib.py
COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python3.8/datalab/fab.py
-COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
COPY ${SRC_PATH}project/templates/locations/ /root/locations/
diff --git a/infrastructure-provisioning/src/general/files/os/debian/sources.list b/infrastructure-provisioning/src/general/files/os/debian/sources.list
deleted file mode 100644
index 3b3e80a..0000000
--- a/infrastructure-provisioning/src/general/files/os/debian/sources.list
+++ /dev/null
@@ -1,56 +0,0 @@
-# *****************************************************************************
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# ******************************************************************************
-
-# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
-# newer versions of the distribution.
-
-deb http://mirrors.aliyun.com/ubuntu/ xenial main restricted
-deb-src http://mirrors.aliyun.com/ubuntu/ xenial main restricted
-
-## Major bug fix updates produced after the final release of the
-## distribution.
-deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted
-deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted
-
-## Uncomment the following two lines to add software from the 'universe'
-## repository.
-## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
-## team. Also, please note that software in universe WILL NOT receive any
-## review or updates from the Ubuntu security team.
-deb http://mirrors.aliyun.com/ubuntu/ xenial universe
-deb-src http://mirrors.aliyun.com/ubuntu/ xenial universe
-deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
-deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
-
-## N.B. software from this repository may not have been tested as
-## extensively as that contained in the main release, although it includes
-## newer versions of some applications which may provide useful features.
-## Also, please note that software in backports WILL NOT receive any review
-## or updates from the Ubuntu security team.
-# deb http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted
-# deb-src http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted
-
-deb http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted
-deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted
-deb http://mirrors.aliyun.com/ubuntu/ xenial-security universe
-deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security universe
-# deb http://mirrors.aliyun.com/ubuntu/ xenial-security multiverse
-# deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security multiverse
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/files/os/redhat/sources.list b/infrastructure-provisioning/src/general/files/os/redhat/sources.list
deleted file mode 100644
index fe5549c..0000000
--- a/infrastructure-provisioning/src/general/files/os/redhat/sources.list
+++ /dev/null
@@ -1,83 +0,0 @@
-# *****************************************************************************
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# ******************************************************************************
-
-# CentOS-Base.repo
-#
-# The mirror system uses the connecting IP address of the client and the
-# update status of each mirror to pick mirrors that are updated to and
-# geographically close to the client. You should use this for CentOS updates
-# unless you are manually picking other mirrors.
-#
-# If the mirrorlist= does not work for you, as a fall back you can try the
-# remarked out baseurl= line instead.
-#
-#
-
-[base]
-name=CentOS-7 - Base - mirrors.aliyun.com
-failovermethod=priority
-baseurl=http://mirrors.aliyun.com/centos/7/os/$basearch/
- http://mirrors.aliyuncs.com/centos/7/os/$basearch/
-#mirrorlist=http://mirrorlist.centos.org/?release=7&arch=$basearch&repo=os
-gpgcheck=1
-gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
-
-#released updates
-[updates]
-name=CentOS-7 - Updates - mirrors.aliyun.com
-failovermethod=priority
-baseurl=http://mirrors.aliyun.com/centos/7/updates/$basearch/
- http://mirrors.aliyuncs.com/centos/7/updates/$basearch/
-#mirrorlist=http://mirrorlist.centos.org/?release=7&arch=$basearch&repo=updates
-gpgcheck=1
-gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
-
-#additional packages that may be useful
-[extras]
-name=CentOS-7 - Extras - mirrors.aliyun.com
-failovermethod=priority
-baseurl=http://mirrors.aliyun.com/centos/7/extras/$basearch/
- http://mirrors.aliyuncs.com/centos/7/extras/$basearch/
-#mirrorlist=http://mirrorlist.centos.org/?release=7&arch=$basearch&repo=extras
-gpgcheck=1
-gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
-
-#additional packages that extend functionality of existing packages
-[centosplus]
-name=CentOS-7 - Plus - mirrors.aliyun.com
-failovermethod=priority
-baseurl=http://mirrors.aliyun.com/centos/7/centosplus/$basearch/
- http://mirrors.aliyuncs.com/centos/7/centosplus/$basearch/
-#mirrorlist=http://mirrorlist.centos.org/?release=7&arch=$basearch&repo=centosplus
-gpgcheck=1
-enabled=0
-gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
-
-#contrib - packages by Centos Users
-[contrib]
-name=CentOS-7 - Contrib - mirrors.aliyun.com
-failovermethod=priority
-baseurl=http://mirrors.aliyun.com/centos/7/contrib/$basearch/
- http://mirrors.aliyuncs.com/centos/7/contrib/$basearch/
-#mirrorlist=http://mirrorlist.centos.org/?release=7&arch=$basearch&repo=contrib
-gpgcheck=1
-enabled=0
-gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
index 29b504b..e62d301 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
@@ -156,12 +156,7 @@ def renew_gpg_key():
sys.exit(1)
-def change_pkg_repos():
- if not exists(datalab.fab.conn,'/tmp/pkg_china_ensured'):
- datalab.fab.conn.put('/root/files/sources.list', '/tmp/sources.list')
- datalab.fab.conn.sudo('mv /tmp/sources.list /etc/apt/sources.list')
- manage_pkg('update', 'remote', '')
- datalab.fab.conn.sudo('touch /tmp/pkg_china_ensured')
+
def find_java_path_remote():
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
index 6b432fd..15d51c5 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
@@ -89,13 +89,6 @@ def ensure_pkg(user, requisites='git vim gcc python-devel openssl-devel nmap lib
sys.exit(1)
-def change_pkg_repos():
- if not exists(datalab.fab.conn,'/tmp/pkg_china_ensured'):
- datalab.fab.conn.put('/root/files/sources.list', '/tmp/sources.list')
- datalab.fab.conn.sudo('mv /tmp/sources.list /etc/yum.repos.d/CentOS-Base-aliyun.repo')
- datalab.fab.conn.sudo('touch /tmp/pkg_china_ensured')
-
-
def find_java_path_remote():
java_path = datalab.fab.conn.sudo("alternatives --display java | grep 'slave jre: ' | awk '{print $3}'").stdout.replace('\n','')
return java_path
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py b/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
index 4d542b4..b265226 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
@@ -87,12 +87,12 @@ if __name__ == "__main__":
print('[INSTALLING KERNELS INTO SPECIFIED NOTEBOOK]')
params = "--bucket {} --cluster_name {} --emr_version {} --keyfile {} --notebook_ip {} --region {} " \
"--emr_excluded_spark_properties {} --project_name {} --os_user {} --edge_hostname {} " \
- "--proxy_port {} --scala_version {} --application {} --pip_mirror {}" \
+ "--proxy_port {} --scala_version {} --application {}" \
.format(notebook_config['bucket_name'], notebook_config['cluster_name'], os.environ['emr_version'],
notebook_config['key_path'], notebook_config['notebook_ip'], os.environ['aws_region'],
os.environ['emr_excluded_spark_properties'], os.environ['project_name'],
os.environ['conf_os_user'], edge_instance_hostname, '3128', os.environ['notebook_scala_version'],
- os.environ['application'], os.environ['conf_pypi_mirror'])
+ os.environ['application'])
try:
subprocess.run("~/scripts/{}_{}.py {}".format(application, 'install_dataengine-service_kernels', params), shell=True, check=True)
datalab.actions_lib.remove_emr_tag(notebook_config['cluster_id'], ['State'])
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index bee2e3f..6ce0672 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@ -37,7 +37,8 @@ if __name__ == "__main__":
local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
- filename=local_log_filepath)
+ filename=local_log_filepath,
+ handlers=[logging.StreamHandler()])
ssn_conf = dict()
ssn_conf['instance'] = 'ssn'
ssn_conf['pre_defined_vpc'] = False
@@ -59,7 +60,6 @@ if __name__ == "__main__":
try:
logging.info('[DERIVING NAMES]')
- print('[DERIVING NAMES]')
ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] = datalab.fab.replace_multi_symbols(
os.environ['conf_service_base_name'][:20], '-', True)
ssn_conf['role_name'] = '{}-ssn-role'.format(ssn_conf['service_base_name'])
@@ -101,7 +101,6 @@ if __name__ == "__main__":
try:
ssn_conf['pre_defined_vpc'] = True
logging.info('[CREATE VPC AND ROUTE TABLE]')
- print('[CREATE VPC AND ROUTE TABLE]')
params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {} --vpc_name {}".format(
ssn_conf['vpc_cidr'], ssn_conf['region'], ssn_conf['tag_name'], ssn_conf['service_base_name'],
ssn_conf['vpc_name'])
@@ -127,7 +126,6 @@ if __name__ == "__main__":
try:
ssn_conf['pre_defined_vpc2'] = True
logging.info('[CREATE SECONDARY VPC AND ROUTE TABLE]')
- print('[CREATE SECONDARY VPC AND ROUTE TABLE]')
params = "--vpc {} --region {} --infra_tag_name {} --infra_tag_value {} --secondary " \
"--vpc_name {}".format(ssn_conf['vpc2_cidr'], ssn_conf['region'], ssn_conf['tag2_name'],
ssn_conf['service_base_name'], ssn_conf['vpc2_name'])
@@ -154,7 +152,6 @@ if __name__ == "__main__":
try:
ssn_conf['pre_defined_subnet'] = True
logging.info('[CREATE SUBNET]')
- print('[CREATE SUBNET]')
params = "--vpc_id {0} --username {1} --infra_tag_name {2} --infra_tag_value {3} --prefix {4} " \
"--ssn {5} --zone {6} --subnet_name {7}".format(
os.environ['aws_vpc_id'], 'ssn', ssn_conf['tag_name'],ssn_conf['service_base_name'], '20',
@@ -193,7 +190,6 @@ if __name__ == "__main__":
except KeyError:
try:
logging.info('[CREATE PEERING CONNECTION]')
- print('[CREATE PEERING CONNECTION]')
os.environ['aws_peering_id'] = datalab.actions_lib.create_peering_connection(
os.environ['aws_vpc_id'], os.environ['aws_vpc2_id'], ssn_conf['service_base_name'])
print('PEERING CONNECTION ID:' + os.environ['aws_peering_id'])
@@ -226,7 +222,6 @@ if __name__ == "__main__":
try:
ssn_conf['pre_defined_sg'] = True
logging.info('[CREATE SG FOR SSN]')
- print('[CREATE SG FOR SSN]')
ssn_conf['ingress_sg_rules_template'] = datalab.meta_lib.format_sg([
{
"PrefixListIds": [],
@@ -301,7 +296,6 @@ if __name__ == "__main__":
try:
logging.info('[CREATE ROLES]')
- print('[CREATE ROLES]')
params = "--role_name {} --role_profile_name {} --policy_name {} --policy_file_name {} --region {} " \
"--infra_tag_name {} --infra_tag_value {} --user_tag_value {}".\
format(ssn_conf['role_name'], ssn_conf['role_profile_name'], ssn_conf['policy_name'],
@@ -335,7 +329,6 @@ if __name__ == "__main__":
try:
logging.info('[CREATE ENDPOINT AND ROUTE-TABLE]')
- print('[CREATE ENDPOINT AND ROUTE-TABLE]')
params = "--vpc_id {} --region {} --infra_tag_name {} --infra_tag_value {}".format(
os.environ['aws_vpc_id'], os.environ['aws_region'], ssn_conf['tag_name'], ssn_conf['service_base_name'])
try:
@@ -368,7 +361,6 @@ if __name__ == "__main__":
if os.environ['conf_duo_vpc_enable'] == 'true':
try:
logging.info('[CREATE ENDPOINT AND ROUTE-TABLE FOR NOTEBOOK VPC]')
- print('[CREATE ENDPOINT AND ROUTE-TABLE FOR NOTEBOOK VPC]')
params = "--vpc_id {} --region {} --infra_tag_name {} --infra_tag_value {}".format(
os.environ['aws_vpc2_id'], os.environ['aws_region'], ssn_conf['tag2_name'],
ssn_conf['service_base_name'])
@@ -401,7 +393,6 @@ if __name__ == "__main__":
try:
logging.info('[CREATE SSN INSTANCE]')
- print('[CREATE SSN INSTANCE]')
params = "--node_name {0} --ami_id {1} --instance_type {2} --key_name {3} --security_group_ids {4} " \
"--subnet_id {5} --iam_profile {6} --infra_tag_name {7} --infra_tag_value {8} --instance_class {9} " \
"--primary_disk_size {10}".\
@@ -441,7 +432,6 @@ if __name__ == "__main__":
if ssn_conf['network_type'] == 'public':
try:
logging.info('[ASSOCIATING ELASTIC IP]')
- print('[ASSOCIATING ELASTIC IP]')
ssn_conf['ssn_id'] = datalab.meta_lib.get_instance_by_name(ssn_conf['tag_name'], ssn_conf['instance_name'])
try:
ssn_conf['elastic_ip'] = os.environ['ssn_elastic_ip']
@@ -489,7 +479,6 @@ if __name__ == "__main__":
if 'ssn_hosted_zone_id' in os.environ and 'ssn_hosted_zone_name' in os.environ and 'ssn_subdomain' in os.environ:
try:
logging.info('[CREATING ROUTE53 RECORD]')
- print('[CREATING ROUTE53 RECORD]')
try:
datalab.actions_lib.create_route_53_record(os.environ['ssn_hosted_zone_id'],
os.environ['ssn_hosted_zone_name'],
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
index a79a4c4..7273709 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_notebook_configure_dataengine-service.py
@@ -100,12 +100,11 @@ if __name__ == "__main__":
print('[INSTALLING KERNELS INTO SPECIFIED NOTEBOOK]')
params = "--bucket {} --cluster_name {} --dataproc_version {} --keyfile {} --notebook_ip {} --region {} " \
"--edge_user_name {} --project_name {} --os_user {} --edge_hostname {} --proxy_port {} " \
- "--scala_version {} --application {} --pip_mirror {}" \
+ "--scala_version {} --application {}" \
.format(notebook_config['bucket_name'], notebook_config['cluster_name'], os.environ['dataproc_version'],
notebook_config['key_path'], notebook_config['notebook_ip'], os.environ['gcp_region'],
notebook_config['edge_user_name'], notebook_config['project_name'], os.environ['conf_os_user'],
- edge_instance_hostname, '3128', os.environ['notebook_scala_version'], os.environ['application'],
- os.environ['conf_pypi_mirror'])
+ edge_instance_hostname, '3128', os.environ['notebook_scala_version'], os.environ['application'])
try:
subprocess.run("~/scripts/{}_{}.py {}".format(application, 'install_dataengine-service_kernels', params), shell=True, check=True)
GCPActions.update_dataproc_cluster(notebook_config['cluster_name'], notebook_config['cluster_labels'])
diff --git a/infrastructure-provisioning/src/ssn/fabfile.py b/infrastructure-provisioning/src/ssn/fabfile.py
index 79243cd..1c107b0 100644
--- a/infrastructure-provisioning/src/ssn/fabfile.py
+++ b/infrastructure-provisioning/src/ssn/fabfile.py
@@ -26,8 +26,6 @@ import os
import sys
import traceback
import uuid
-from datalab.fab import *
-from fabric import *
@task
def run(ctx):
@@ -39,14 +37,16 @@ def run(ctx):
ssn_config = dict()
ssn_config['ssn_unique_index'] = str(uuid.uuid4())[:5]
try:
- subprocess.run("~/scripts/{}.py --ssn_unique_index {}".format('ssn_prepare', ssn_config['ssn_unique_index']), shell=True, check=True)
+ subprocess.run("~/scripts/{}.py --ssn_unique_index {}".format('ssn_prepare', ssn_config['ssn_unique_index']),
+ shell=True, check=True)
except Exception as err:
traceback.print_exc()
append_result("Failed preparing SSN node.", str(err))
sys.exit(1)
try:
- subprocess.run("~/scripts/{}.py --ssn_unique_index {}".format('ssn_configure', ssn_config['ssn_unique_index']), shell=True, check=True)
+ subprocess.run("~/scripts/{}.py --ssn_unique_index {}".format('ssn_configure', ssn_config['ssn_unique_index']),
+ shell=True, check=True)
except Exception as err:
traceback.print_exc()
append_result("Failed configuring SSN node.", str(err))
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index aa20a68..8c0e4cb 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -73,11 +73,6 @@ def download_toree():
sys.exit(1)
-def add_china_repository(datalab_path):
- conn.sudo('''bash -c 'cd {1}sources/infrastructure-provisioning/src/base/ && sed -i "/pip install/s/$/ -i https\:\/\/{0}\/simple --trusted-host {0} --timeout 60000/g" Dockerfile' '''.format(os.environ['conf_pypi_mirror'], datalab_path))
- conn.sudo('''bash -c 'cd {}sources/infrastructure-provisioning/src/base/ && sed -i "/pip install/s/jupyter/ipython==5.0.0 jupyter==1.0.0/g" Dockerfile' '''.format(datalab_path))
- conn.sudo('''bash -c 'cd {}sources/infrastructure-provisioning/src/base/ && sed -i "22i COPY general/files/os/debian/sources.list /etc/apt/sources.list" Dockerfile' '''.format(datalab_path))
-
def login_in_gcr(os_user, gcr_creds, odahu_image, datalab_path, cloud_provider):
if gcr_creds != '':
try:
@@ -119,8 +114,6 @@ def build_docker_images(image_list, region, datalab_path):
'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
'/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
- if region == 'cn-north-1':
- add_china_repository(datalab_path)
for image in image_list:
name = image['name']
tag = image['tag']
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org