You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/09/09 10:06:22 UTC
[incubator-datalab] 01/01: Merge branch 'develop' into DATALAB-2409
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2409
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 7c06edf11311f977f53933aee00244f8272739a0
Merge: 645e7b6 46b6956
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Thu Sep 9 13:05:38 2021 +0300
Merge branch 'develop' into DATALAB-2409
# Conflicts:
# infrastructure-provisioning/scripts/deploy_datalab.py
# infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
# infrastructure-provisioning/src/general/lib/os/fab.py
# infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
# infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
# infrastructure-provisioning/src/ssn/scripts/configure_docker.py
.../scripts/deploy_datalab.py | 28 +-
.../scripts/deploy_repository/deploy_repository.py | 2 +-
.../scripts/jenkins/sonar.py | 4 +-
.../scripts/configure_keycloak.py | 16 +-
.../src/base/scripts/install_user_key.py | 2 +-
.../src/dataengine/scripts/configure_dataengine.py | 5 +-
.../src/general/conf/datalab.ini | 8 +
.../files/azure/deeplearning_description.json | 2 +-
.../src/general/lib/aws/actions_lib.py | 7 +-
.../src/general/lib/azure/actions_lib.py | 77 ++-
.../src/general/lib/os/debian/common_lib.py | 215 ++++--
.../src/general/lib/os/fab.py | 17 +-
.../scripts/aws/common_create_role_policy.py | 3 +-
.../scripts/aws/dataengine-service_create.py | 6 +-
.../scripts/aws/dataengine-service_prepare.py | 2 +
.../src/general/scripts/aws/edge_configure.py | 2 +-
.../src/general/scripts/aws/project_prepare.py | 4 +
.../src/general/scripts/aws/ssn_configure.py | 12 +
.../src/general/scripts/aws/ssn_prepare.py | 2 +
.../src/general/scripts/aws/ssn_terminate.py | 38 ++
.../scripts/azure/common_prepare_notebook.py | 35 +-
.../general/scripts/azure/dataengine_configure.py | 4 +-
.../src/general/scripts/azure/edge_configure.py | 2 +-
.../src/general/scripts/azure/ssn_configure.py | 21 +
.../src/general/scripts/azure/ssn_terminate.py | 38 ++
.../src/general/scripts/gcp/edge_configure.py | 2 +-
.../src/general/scripts/gcp/ssn_configure.py | 21 +
.../src/general/scripts/gcp/ssn_terminate.py | 22 +
.../general/scripts/os/common_clean_instance.py | 2 +
.../scripts/os/update_inactivity_on_start.py | 2 +
.../src/ssn/scripts/configure_docker.py | 36 +-
.../src/ssn/scripts/configure_ui.py | 12 +-
.../zeppelin/scripts/configure_zeppelin_node.py | 2 +-
.../com/epam/datalab/dto/ResourceSysBaseDTO.java | 8 +-
.../com/epam/datalab/dto/base/edge/EdgeInfo.java | 7 +-
.../dto/exploratory/ExploratoryActionDTO.java | 1 -
.../response/handlers/EdgeCallbackHandler.java | 3 +-
.../response/handlers/ProjectCallbackHandler.java | 1 -
.../response/handlers/ResourceCallbackHandler.java | 1 -
.../handlers/ResourcesStatusCallbackHandler.java | 21 +-
.../backendapi/domain/NotebookTemplate.java | 5 +-
.../backendapi/resources/ProjectResource.java | 1 +
.../resources/callback/ProjectCallback.java | 6 +-
.../resources/dto/ProjectActionFormDTO.java | 2 +
.../service/impl/ExploratoryServiceImpl.java | 2 +-
.../service/impl/LibraryServiceImpl.java | 6 +-
.../service/impl/ProjectServiceImpl.java | 5 +-
.../configuration/configuration.component.html | 104 ++-
.../configuration/configuration.component.scss | 48 +-
.../configuration/configuration.component.ts | 203 +++---
.../backup-dilog/backup-dilog.component.html | 41 +-
.../backup-dilog/backup-dilog.component.ts | 41 +-
.../management/endpoints/endpoints.component.html | 122 +++-
.../management/endpoints/endpoints.component.scss | 24 +-
.../management/endpoints/endpoints.component.ts | 103 +--
.../manage-environment-dilog.component.html | 99 ++-
.../manage-environment-dilog.component.scss | 8 +-
.../manage-environment-dilog.component.ts | 34 +-
.../management/management-data.service.ts | 6 +-
.../management-grid/management-grid.component.html | 337 +++++++---
.../management-grid/management-grid.component.scss | 164 +++--
.../management-grid/management-grid.component.ts | 116 ++--
.../management/management.component.html | 59 +-
.../management/management.component.ts | 128 ++--
.../administration/management/management.model.ts | 1 -
.../ssn-monitor/ssn-monitor.component.html | 9 +-
.../ssn-monitor/ssn-monitor.component.scss | 8 +-
.../ssn-monitor/ssn-monitor.component.ts | 7 +-
.../administration/project/project-data.service.ts | 7 -
.../project-form/project-form.component.html | 155 ++++-
.../project-form/project-form.component.scss | 8 +-
.../project/project-form/project-form.component.ts | 15 +-
.../project-list/project-list.component.html | 193 +++---
.../project-list/project-list.component.scss | 68 +-
.../project/project-list/project-list.component.ts | 88 +--
.../administration/project/project.component.html | 27 +-
.../administration/project/project.component.ts | 56 +-
.../app/administration/roles/roles.component.html | 137 ++--
.../app/administration/roles/roles.component.scss | 77 ++-
.../app/administration/roles/roles.component.ts | 259 +++++---
.../resources/webapp/src/app/app.routing.module.ts | 153 +++--
.../app/core/directives/click-outside.directive.ts | 9 +-
.../src/app/core/directives/scrollTo.directive.ts | 2 +-
.../src/app/core/interceptors/error.interceptor.ts | 5 +-
.../core/interceptors/http.token.interceptor.ts | 12 +-
...mputationalResourceApplicationTemplate.model.ts | 7 +-
.../models/computationalResourceImage.model.ts | 17 +-
.../app/core/models/resourceShapeTypes.model.ts | 3 +-
.../webapp/src/app/core/pipes/highlight.pipe.ts | 7 +-
.../app/core/pipes/lib-sort-pipe/lib-sort.pipe.ts | 2 +-
.../local-currency-pipe/local-currency.pipe.ts | 4 +-
.../src/app/core/pipes/local-date-pipe/index.ts | 2 +-
.../core/pipes/local-date-pipe/local-date.pipe.ts | 6 +-
.../core/pipes/long-date-pipe/long-date.pipe.ts | 8 +-
.../core/services/applicationSecurity.service.ts | 3 +-
.../services/applicationServiceFacade.service.ts | 5 +-
.../webapp/src/app/core/services/audit.service.ts | 6 +
.../src/app/core/services/authorization.guard.ts | 6 +-
.../app/core/services/bucket-browser.service.ts | 11 +-
.../src/app/core/services/checkParams.guard.ts | 6 +-
.../app/core/services/configutration.service.ts | 3 +-
.../src/app/core/services/healthStatus.service.ts | 14 +-
.../core/services/librariesInstallation.service.ts | 3 +-
.../app/core/services/odahu-deployment.service.ts | 8 +-
.../src/app/core/services/progress-bar.service.ts | 5 +-
.../src/app/core/services/project.service.ts | 3 +-
.../src/app/core/services/userAccessKey.service.ts | 15 +-
.../src/app/core/services/userResource.service.ts | 15 +-
.../webapp/src/app/core/util/checkUtils.ts | 1 +
.../webapp/src/app/core/util/compareUtils.ts | 3 -
.../webapp/src/app/core/util/dateUtils.ts | 4 +-
.../webapp/src/app/core/util/http-status-codes.ts | 106 +--
.../webapp/src/app/core/util/sortUtils.ts | 3 +-
.../webapp/src/app/login/login.component.css | 38 +-
.../audit/audit-grid/audit-grid.component.html | 150 ++++-
.../audit/audit-grid/audit-grid.component.scss | 70 +-
.../audit/audit-grid/audit-grid.component.ts | 70 +-
.../audit-toolbar/audit-toolbar.component.scss | 11 +-
.../audit/audit-toolbar/audit-toolbar.component.ts | 9 +-
.../src/app/reports/audit/audit.component.ts | 7 +-
.../reporting-grid/reporting-grid.component.html | 256 ++++++--
.../reporting-grid/reporting-grid.component.scss | 173 ++---
.../reporting-grid/reporting-grid.component.ts | 74 ++-
.../app/reports/reporting/reporting.component.ts | 39 +-
.../src/app/reports/reporting/reporting.module.ts | 4 +-
.../reporting/toolbar/toolbar.component.html | 18 +-
.../reporting/toolbar/toolbar.component.scss | 11 +-
.../reports/reporting/toolbar/toolbar.component.ts | 16 +-
.../bucket-browser/bucket-browser.component.html | 276 ++++----
.../bucket-browser/bucket-browser.component.scss | 502 +++++++-------
.../bucket-browser/bucket-browser.component.ts | 165 ++---
.../bucket-browser/bucket-browser.module.ts | 46 +-
.../bucket-confirmation-dialog.component.html | 98 ++-
.../bucket-confirmation-dialog.component.scss | 93 +--
.../bucket-confirmation-dialog.component.ts | 13 +-
.../bucket-browser/bucket-data.service.ts | 83 ++-
.../buckets-tree/bucket-tree.component.html | 26 +-
.../buckets-tree/bucket-tree.component.scss | 44 +-
.../buckets-tree/bucket-tree.component.ts | 4 +-
.../folder-tree/folder-tree.component.html | 56 +-
.../folder-tree/folder-tree.component.scss | 45 +-
.../folder-tree/folder-tree.component.ts | 108 +--
.../bucket-browser/upload-window.component.scss | 94 +--
.../cluster-details/cluster-details.component.html | 123 ++--
.../cluster-details/cluster-details.component.scss | 6 +-
.../cluster-details/cluster-details.component.ts | 38 +-
.../computational/cluster-details/index.ts | 10 +-
...utational-resource-create-dialog.component.html | 378 ++++++-----
...utational-resource-create-dialog.component.scss | 43 +-
...mputational-resource-create-dialog.component.ts | 33 +-
.../computational-resources-list.component.html | 46 +-
.../computational-resources-list.component.scss | 208 +++---
.../computational-resources-list.component.ts | 2 +-
.../ami-create-dialog.component.html | 20 +-
.../ami-create-dialog.component.scss | 4 +-
.../ami-create-dialog.component.ts | 15 +-
.../cost-details-dialog.component.html | 41 +-
.../cost-details-dialog.component.scss | 18 +-
.../create-environment.component.html | 311 +++++----
.../create-environment.component.scss | 11 +-
.../create-environment.component.ts | 31 +-
.../detail-dialog/detail-dialog.component.html | 378 ++++++-----
.../detail-dialog/detail-dialog.component.scss | 41 +-
.../detail-dialog/detail-dialog.component.ts | 53 +-
.../install-libraries.component.html | 730 +++++++++++----------
.../install-libraries.component.scss | 118 ++--
.../install-libraries.component.ts | 68 +-
.../install-libraries/install-libraries.model.ts | 15 +-
.../libraries-info.component.scss | 43 +-
.../manage-ungit/manage-ungit.component.html | 78 ++-
.../manage-ungit/manage-ungit.component.scss | 18 +-
.../src/app/resources/resources-grid/index.ts | 38 +-
.../resources-grid/resources-grid.component.html | 229 ++++---
.../resources-grid/resources-grid.component.scss | 119 ++--
.../resources-grid/resources-grid.component.ts | 123 ++--
.../resources-grid/resources-grid.model.ts | 4 +-
.../src/app/resources/resources.component.html | 42 +-
.../src/app/resources/resources.component.ts | 24 +-
.../webapp/src/app/resources/resources.module.ts | 30 +-
.../resources/scheduler/scheduler.component.html | 317 +++++----
.../resources/scheduler/scheduler.component.scss | 18 +-
.../app/resources/scheduler/scheduler.component.ts | 21 +-
.../src/app/shared/bubble/bubble.component.css | 4 +-
.../src/app/shared/bubble/bubble.component.ts | 15 +-
.../dropdown-list/dropdown-list.component.html | 8 +-
.../shared/form-controls/dropdowns.component.scss | 36 +-
.../multi-level-select-dropdown.component.html | 167 +++--
.../multi-level-select-dropdown.component.scss | 72 +-
.../multi-level-select-dropdown.component.ts | 17 +-
.../multi-select-dropdown.component.html | 51 +-
.../inform-message/inform-message.component.css | 10 +-
.../inform-message/inform-message.component.ts | 12 +-
.../src/app/shared/navbar/navbar.component.html | 129 ++--
.../src/app/shared/navbar/navbar.component.scss | 56 +-
.../src/app/shared/navbar/navbar.component.ts | 17 +-
.../src/app/shared/time-picker/ticker.component.ts | 9 +-
.../app/shared/time-picker/time-cover.component.ts | 2 +-
.../shared/time-picker/time-picker.component.scss | 44 +-
.../shared/time-picker/time-picker.component.ts | 20 +-
.../src/app/webterminal/webterminal.component.ts | 3 +-
.../webapp/src/assets/styles/_dialogs.scss | 139 ++--
.../webapp/src/assets/styles/_general.scss | 161 +++--
.../resources/webapp/src/assets/styles/_reset.scss | 12 +-
.../resources/webapp/src/assets/styles/_theme.scss | 271 ++++----
.../src/main/resources/webapp/src/styles.scss | 116 ++--
.../backendapi/resources/ProjectResourceTest.java | 4 +-
206 files changed, 6951 insertions(+), 4797 deletions(-)
diff --cc infrastructure-provisioning/scripts/deploy_datalab.py
index 5932d34,7e9541d..efeeaca
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@@ -63,194 -24,165 +63,196 @@@
import argparse
import os
import subprocess
-from fabric import *
-from invoke import task
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--conf_service_base_name', type=str, help='unique name for DataLab environment')
-parser.add_argument('--conf_network_type', type=str, default='',
- help='Define in which network DataLab will be deployed. '
- 'Possible options: public|private')
-parser.add_argument('--conf_vpc_cidr', type=str, default='', help='CIDR of VPC')
-parser.add_argument('--conf_vpc2_cidr', type=str, default='', help='CIDR of secondary VPC')
-parser.add_argument('--conf_allowed_ip_cidr', type=str, default='', help='Comma-separated CIDR of IPs which will have '
- 'access to SSN')
-parser.add_argument('--conf_user_subnets_range', type=str, default='', help='Range of subnets which will be using for '
- 'users environments. For example: '
- '10.10.0.0/24 - 10.10.10.0/24')
-parser.add_argument('--conf_private_subnet_prefix', type=str, default='24', help='Private subnet prefix')
-parser.add_argument('--conf_additional_tags', type=str, default='', help='Additional tags in format '
- '"Key1:Value1;Key2:Value2"')
-parser.add_argument('--conf_image_enabled', type=str, default='', help='Enable or Disable creating image at first time')
-parser.add_argument('--aws_user_predefined_s3_policies', type=str, default='', help='Predefined policies for users '
- 'instances')
-parser.add_argument('--aws_access_key', type=str, default='', help='AWS Access Key ID')
-parser.add_argument('--aws_secret_access_key', type=str, default='', help='AWS Secret Access Key')
-parser.add_argument('--aws_region', type=str, default='', help='AWS region')
-parser.add_argument('--aws_zone', type=str, default='', help='AWS zone')
-parser.add_argument('--azure_region', type=str, default='', help='Azure region')
-parser.add_argument('--gcp_region', type=str, default='', help='GCP region')
-parser.add_argument('--gcp_zone', type=str, default='', help='GCP zone')
-parser.add_argument('--conf_os_family', type=str, default='',
- help='Operating system type. Available options: debian, redhat')
-parser.add_argument('--conf_cloud_provider', type=str, default='',
- help='Where DataLab should be deployed. Available options: aws, azure, gcp')
-parser.add_argument('--ssn_hosted_zone_name', type=str, default='', help='Name of hosted zone')
-parser.add_argument('--ssn_hosted_zone_id', type=str, default='', help='ID of hosted zone')
-parser.add_argument('--ssn_subdomain', type=str, default='', help='Subdomain name')
-parser.add_argument('--ssn_assume_role_arn', type=str, default='', help='Role ARN for creating Route53 record in '
- 'different AWS account')
-parser.add_argument('--ssl_cert_path', type=str, default='', help='Full path to SSL certificate')
-parser.add_argument('--ssl_key_path', type=str, default='', help='Full path to key for SSL certificate')
-parser.add_argument('--aws_vpc_id', type=str, default='', help='AWS VPC ID')
-parser.add_argument('--conf_duo_vpc_enable', type=str, default='false', help='Duo VPC scheme enable(true|false)')
-parser.add_argument('--aws_vpc2_id', type=str, default='', help='Secondary AWS VPC ID')
-parser.add_argument('--aws_peering_id', type=str, default='', help='Amazon peering connection id')
-parser.add_argument('--azure_vpc_name', type=str, default='', help='Azure VPC Name')
-parser.add_argument('--gcp_vpc_name', type=str, default='', help='GCP VPC Name')
-parser.add_argument('--aws_subnet_id', type=str, default='', help='AWS Subnet ID')
-parser.add_argument('--azure_subnet_name', type=str, default='', help='Azure Subnet Name')
-parser.add_argument('--gcp_subnet_name', type=str, default='', help='GCP Subnet Name')
-parser.add_argument('--aws_security_groups_ids', type=str, default='', help='One of more comma-separated Security '
- 'groups IDs for SSN')
-parser.add_argument('--azure_security_group_name', type=str, default='', help='One Security'
- 'group name for SSN')
-parser.add_argument('--azure_edge_security_group_name', type=str, default='', help='One Security '
- 'group name for Edge node')
-parser.add_argument('--gcp_firewall_name', type=str, default='', help='One of more comma-separated GCP Firewall rules '
- 'for SSN')
-parser.add_argument('--key_path', type=str, default='', help='Path to admin key (WITHOUT KEY NAME)')
-parser.add_argument('--conf_key_name', type=str, default='', help='Admin key name (WITHOUT ".pem")')
-parser.add_argument('--workspace_path', type=str, default='', help='Admin key name (WITHOUT ".pem")')
-parser.add_argument('--conf_tag_resource_id', type=str, default='datalab', help='The name of user tag')
-parser.add_argument('--conf_billing_tag', type=str, default='datalab', help='Billing tag')
-parser.add_argument('--aws_ssn_instance_size', type=str, default='t2.large', help='The SSN instance shape')
-parser.add_argument('--azure_ssn_instance_size', type=str, default='Standard_DS2_v2', help='The SSN instance shape')
-parser.add_argument('--gcp_ssn_instance_size', type=str, default='n1-standard-2', help='The SSN instance shape')
-parser.add_argument('--aws_account_id', type=str, default='', help='The ID of Amazon account')
-parser.add_argument('--aws_billing_bucket', type=str, default='', help='The name of S3 bucket where billing reports '
- 'will be placed.')
-parser.add_argument('--aws_job_enabled', type=str, default='false', help='Billing format. Available options: '
- 'true (aws), false(epam)')
-parser.add_argument('--aws_report_path', type=str, default='', help='The path to billing reports directory in S3 '
- 'bucket')
-parser.add_argument('--aws_permissions_boundary_arn', type=str, default='', help='Permission boundary to be attached to new roles')
-parser.add_argument('--azure_resource_group_name', type=str, default='', help='Name of Resource group in Azure')
-parser.add_argument('--azure_auth_path', type=str, default='', help='Full path to Azure credentials JSON file')
-parser.add_argument('--azure_datalake_enable', type=str, default='', help='Provision DataLake storage account')
-parser.add_argument('--azure_ad_group_id', type=str, default='', help='ID of Azure AD group')
-parser.add_argument('--azure_offer_number', type=str, default='', help='Azure offer number')
-parser.add_argument('--azure_currency', type=str, default='', help='Azure currency code')
-parser.add_argument('--azure_locale', type=str, default='', help='Azure locale')
-parser.add_argument('--azure_application_id', type=str, default='', help='Azure login application ID')
-parser.add_argument('--azure_validate_permission_scope', type=str, default='true', help='Azure permission scope '
- 'validation(true|false).')
-parser.add_argument('--azure_oauth2_enabled', type=str, default='false', help='Using OAuth2 for logging in DataLab')
-parser.add_argument('--azure_region_info', type=str, default='', help='Azure region info')
-parser.add_argument('--azure_source_vpc_name', type=str, default='', help='Azure VPC source Name')
-parser.add_argument('--azure_source_resource_group_name', type=str, default='', help='Azure source resource group')
-parser.add_argument('--gcp_project_id', type=str, default='', help='The project ID in Google Cloud Platform')
-parser.add_argument('--gcp_service_account_path', type=str, default='', help='The project ID in Google Cloud Platform')
-parser.add_argument('--datalab_id', type=str, default="'resource_tags_user_user_tag'",
- help='Column name in report file that contains '
- 'datalab id tag')
-parser.add_argument('--usage_date', type=str, default='line_item_usage_start_date',
- help='Column name in report file that contains '
- 'usage date tag')
-parser.add_argument('--product', type=str, default='product_product_name',
- help='Column name in report file that contains '
- 'product name tag')
-parser.add_argument('--usage_type', type=str, default='line_item_usage_type',
- help='Column name in report file that contains '
- 'usage type tag')
-parser.add_argument('--usage', type=str, default='line_item_usage_amount',
- help='Column name in report file that contains '
- 'usage tag')
-parser.add_argument('--cost', type=str, default='line_item_blended_cost',
- help='Column name in report file that contains cost tag')
-parser.add_argument('--resource_id', type=str, default='line_item_resource_id',
- help='Column name in report file that contains '
- 'datalab resource id tag')
-parser.add_argument('--ldap_hostname', type=str, default='localhost', help='Ldap instance hostname')
-parser.add_argument('--ldap_dn', type=str, default='dc=example,dc=com',
- help='Ldap distinguished name')
-parser.add_argument('--ldap_ou', type=str, default='ou=People', help='Ldap organisation unit')
-parser.add_argument('--ldap_service_username', type=str, default='cn=service-user', help='Ldap service user name')
-parser.add_argument('--ldap_service_password', type=str, default='service-user-password',
- help='Ldap password for admin user')
-parser.add_argument('--keycloak_realm_name', type=str, default='datalab', help='Keycloak Realm name')
-parser.add_argument('--keycloak_auth_server_url', type=str, default='datalab', help='Keycloak auth server URL')
-parser.add_argument('--keycloak_client_name', type=str, help='Keycloak client name')
-parser.add_argument('--keycloak_client_secret', type=str, help='Keycloak client secret')
-parser.add_argument('--keycloak_user', type=str, default='datalab', help='Keycloak user')
-parser.add_argument('--keycloak_user_password', type=str, default='keycloak-user-password',
- help='Keycloak user password')
-parser.add_argument('--tags', type=str, default='line_item_operation,line_item_line_item_description',
- help='Column name in report file that '
- 'contains tags')
-parser.add_argument('--billing_dataset_name', type=str, default='', help='Name of GCP dataset (BigQuery service)'
- ' for billing')
-parser.add_argument('--default_endpoint_name', type=str, default='local', help='Name of localhost provisioning service,'
- 'that created by default')
-parser.add_argument('--conf_stepcerts_enabled', type=str, default='false', help='Enable or disable step certificates')
-parser.add_argument('--conf_stepcerts_root_ca', type=str, default='', help='Step root CA')
-parser.add_argument('--conf_stepcerts_kid', type=str, default='', help='Step KID')
-parser.add_argument('--conf_stepcerts_kid_password', type=str, default='', help='Step KID password')
-parser.add_argument('--conf_stepcerts_ca_url', type=str, default='', help='Step CA URL')
-parser.add_argument('--conf_letsencrypt_enabled', type=str, default='false',
- help='Enable or disable Let`s Encrypt certificates')
-parser.add_argument('--conf_repository_user', type=str, default='',
- help='user to access repository (used for jars download)')
-parser.add_argument('--conf_release_tag', type=str, default='2.5',
- help='tag used for jars download')
-parser.add_argument('--conf_repository_pass', type=str, default='',
- help='password to access repository (used for jars download)')
-parser.add_argument('--conf_repository_address', type=str, default='',
- help='address to access repository (used for jars download)')
-parser.add_argument('--conf_repository_port', type=str, default='',
- help='port to access repository (used for jars download)')
-parser.add_argument('--conf_download_jars', type=str, default='false',
- help='whether to download jars from repository (True) or to build from sources (False)')
-parser.add_argument('--conf_download_docker_images', type=str, default='false',
- help='whether to download docker images from repository (True) or to build from sources (False)')
-parser.add_argument('--conf_letsencrypt_domain_name', type=str, default='', help='Domain names to apply. '
- 'For multiple domains enter a comma separated list of domains as a parameter'
- 'ssn.domain_name will be used for ssn_node, DNS A record have to exist during deployment')
-parser.add_argument('--conf_letsencrypt_email', type=str, default='', help='Email that will be entered during '
- 'certificate obtaining and can be user for urgent renewal and security notices. '
- 'Use comma to register multiple emails, e.g. u1@example.com,u2@example.com.')
-parser.add_argument('--action', required=True, type=str, default='', choices=['build', 'deploy', 'create', 'terminate'],
- help='Available options: build, deploy, create, terminate')
-args = parser.parse_args()
+import sys
+
+BOOL_CHOICES_LIST = ['true', 'false']
+OS_DISTRO_LIST = ['debian', 'redhat']
+NETWORK_TYPE_LIST = ['public', 'private']
+
+
+def build_parser():
+ parser = argparse.ArgumentParser(description='DataLab Self-Service Node deployment',
+ prog='deploy_datalab')
+ # optional arguments
+ parser.add_argument('--conf_network_type', type=str, default='public',
+ help='''Type of network. Define in which network DataLab will be deployed.
+ (valid choices: %s)''' % NETWORK_TYPE_LIST,
+ choices=NETWORK_TYPE_LIST)
+ parser.add_argument('--conf_vpc_cidr', type=str, default='172.31.0.0/16', help='CIDR of VPC')
+ parser.add_argument('--conf_vpc2_cidr', type=str, help='CIDR of secondary VPC')
+ parser.add_argument('--conf_allowed_ip_cidr', type=str, default='0.0.0.0/0',
+ help='Comma-separated CIDR of IPs which will have access to SSN')
+ parser.add_argument('--conf_user_subnets_range', type=str,
+ help='''Range of subnets which will be using for users environments.
+ For example: 10.10.0.0/24 - 10.10.10.0/24''')
+ parser.add_argument('--conf_private_subnet_prefix', type=str, default='24', help='Private subnet prefix')
+ parser.add_argument('--conf_additional_tags', type=str,
+ help='Additional tags in format "Key1:Value1;Key2:Value2"')
+ parser.add_argument('--conf_image_enabled', type=str,
+ help='Enable or Disable creating image at first time')
+ parser.add_argument('--conf_os_family', type=str, default='debian', choices=OS_DISTRO_LIST,
+ help='Operating system distribution. (valid choices: %s)' % OS_DISTRO_LIST)
+ parser.add_argument('--ssn_hosted_zone_name', type=str, help='Name of hosted zone')
+ parser.add_argument('--ssn_hosted_zone_id', type=str, help='ID of hosted zone')
+ parser.add_argument('--ssn_subdomain', type=str, help='Subdomain name')
+ parser.add_argument('--ssl_cert_path', type=str, help='Full path to SSL certificate')
+ parser.add_argument('--ssl_key_path', type=str, help='Full path to key for SSL certificate')
+ parser.add_argument('--workspace_path', type=str, default='', help='Docker workspace path')
+ parser.add_argument('--conf_tag_resource_id', type=str, default='datalab', help='The name of user tag')
+ parser.add_argument('--conf_billing_tag', type=str, default='datalab', help='Billing tag')
+ parser.add_argument('--datalab_id', type=str, default='resource_tags_user_user_tag',
+ help='Column name in report file that contains datalab id tag')
+ parser.add_argument('--usage_date', type=str, default='line_item_usage_start_date',
+ help='Column name in report file that contains usage date tag')
+ parser.add_argument('--product', type=str, default='product_product_name',
+ help='Column name in report file that contains product name tag')
+ parser.add_argument('--usage_type', type=str, default='line_item_usage_type',
+ help='Column name in report file that contains usage type tag')
+ parser.add_argument('--usage', type=str, default='line_item_usage_amount',
+ help='Column name in report file that contains usage tag')
+ parser.add_argument('--cost', type=str, default='line_item_blended_cost',
+ help='Column name in report file that contains cost tag')
+ parser.add_argument('--resource_id', type=str, default='line_item_resource_id',
+ help='Column name in report file that contains datalab resource id tag')
+
+ parser.add_argument('--tags', type=str, default='line_item_operation,line_item_line_item_description',
+ help='Column name in report file that contains tags')
+ parser.add_argument('--conf_stepcerts_enabled', type=str, default='false',
+ help='Enable or disable step certificates. (valid choices: %s)' % BOOL_CHOICES_LIST,
+ choices=BOOL_CHOICES_LIST)
+ parser.add_argument('--conf_stepcerts_root_ca', type=str, help='Step root CA')
+ parser.add_argument('--conf_stepcerts_kid', type=str, help='Step KID')
+ parser.add_argument('--conf_stepcerts_kid_password', type=str, help='Step KID password')
+ parser.add_argument('--conf_stepcerts_ca_url', type=str, help='Step CA URL')
+ parser.add_argument('--conf_letsencrypt_enabled', type=str, default='false',
+ help='Enable or disable Let`s Encrypt certificates. (valid choices: %s)' % BOOL_CHOICES_LIST,
+ choices=BOOL_CHOICES_LIST)
+ parser.add_argument('--conf_letsencrypt_domain_name', type=str,
+ help='''Domain names to apply. For multiple domains enter a comma separated list of domains
+ as a parameter. ssn.domain_name will be used for ssn_node,DNS A record have to exist during deployment''')
+ parser.add_argument('--conf_letsencrypt_email', type=str, help='''Email that will be entered during
+ certificate obtaining and can be user for urgent renewal and security notices. Use comma to register
+ multiple emails, e.g. u1@example.com,u2@example.com.''')
+ parser.add_argument('--conf_repository_user', type=str, default='',
+ help='user to access repository (used for jars download)')
+ parser.add_argument('--conf_release_tag', type=str, default='2.5',
+ help='tag used for jars download')
+ parser.add_argument('--conf_repository_pass', type=str, default='',
+ help='password to access repository (used for jars download)')
+ parser.add_argument('--conf_repository_address', type=str, default='',
+ help='address to access repository (used for jars download)')
+ parser.add_argument('--default_endpoint_name', type=str, default='local',
+ help='Name of localhost provisioning service, that created by default')
+
+ required_args = parser.add_argument_group('Required arguments')
+ required_args.add_argument('--conf_service_base_name', type=str,
+ help='Unique name for DataLab environment', required=True)
+ required_args.add_argument('--action', type=str, help='Action to perform',
+ choices=['build', 'deploy', 'create', 'terminate'], required=True)
+ required_args.add_argument('--key_path', type=str, help='Path to admin key (WITHOUT KEY NAME)', required=True)
+ required_args.add_argument('--conf_key_name', type=str, help='Admin key name (WITHOUT ".pem")', required=True)
+ required_args.add_argument('--keycloak_auth_server_url', type=str, default='datalab',
+ help='Keycloak auth server URL', required=True)
+ required_args.add_argument('--keycloak_realm_name', type=str, help='Keycloak Realm name', required=True)
+ required_args.add_argument('--keycloak_client_name', type=str, default='datalab',
+ help='Keycloak client name', required=True)
+ required_args.add_argument('--keycloak_client_secret', type=str, default='datalab',
+ help='Keycloak client secret', required=True)
+ required_args.add_argument('--keycloak_user', type=str, default='datalab', help='Keycloak user', required=True)
+ required_args.add_argument('--keycloak_user_password', type=str, default='keycloak-user-password',
+ help='Keycloak user password', required=True)
+
+
+ # subparsers
+ subparsers = parser.add_subparsers(dest='conf_cloud_provider', required=True, help='sub-command help',
+ description='''These are the subcommands for deploying resources
+ in a specific cloud provider''')
+
+ # --------- aws subcommand ----------------------
+ aws_parser = subparsers.add_parser('aws')
+ aws_parser.add_argument('--aws_user_predefined_s3_policies', type=str,
+ help='Predefined policies for users instances')
+ aws_parser.add_argument('--aws_access_key', type=str,
+ help='''AWS Access Key ID. reuqired in case of deployment with IAM user DataLab
+ deployment script is executed on local machine and uses
+ IAM user permissions to create resources in AWS.''')
+ aws_parser.add_argument('--aws_secret_access_key', type=str, help='AWS Secret Access Key')
+ aws_parser.add_argument('--aws_ssn_instance_size', type=str, default='t2.large',
+ help='The SSN instance shape')
+ aws_parser.add_argument('--ssn_assume_role_arn', type=str,
+ help='Role ARN for creating Route53 record in different AWS account')
+ aws_parser.add_argument('--aws_vpc_id', type=str, help='AWS VPC ID')
+ aws_parser.add_argument('--conf_duo_vpc_enable', type=str, default='false',
+ help='Duo VPC scheme enable. (valid choices: %s)' % BOOL_CHOICES_LIST,
+ choices=BOOL_CHOICES_LIST)
+ aws_parser.add_argument('--aws_vpc2_id', type=str, help='Secondary AWS VPC ID')
+ aws_parser.add_argument('--aws_peering_id', type=str, help='Amazon peering connection id')
+ aws_parser.add_argument('--aws_subnet_id', type=str, help='AWS Subnet ID')
+ aws_parser.add_argument('--aws_security_groups_ids', type=str,
+ help='One of more comma-separated Security groups IDs for SSN')
+ aws_parser.add_argument('--aws_billing_bucket', type=str,
+ help='The name of S3 bucket where billing reports will be placed.')
+ aws_parser.add_argument('--aws_job_enabled', type=str, default='false', choices=BOOL_CHOICES_LIST,
+ help='Billing format. (valid choices: %s)' % BOOL_CHOICES_LIST)
+ aws_parser.add_argument('--aws_report_path', type=str, help='The path to billing reports directory in S3 bucket')
++ aws_parser.add_argument('--aws_permissions_boundary_arn', type=str, default='',
++ help='Permission boundary to be attached to new roles')
+
+ aws_required_args = aws_parser.add_argument_group('Required arguments')
+ aws_required_args.add_argument('--aws_region', type=str, required=True, help='AWS region')
+ aws_required_args.add_argument('--aws_zone', type=str, required=True, help='AWS zone')
+ aws_required_args.add_argument('--aws_account_id', type=str, required=True, help='The ID of Amazon account')
+
+ # --------azure subcommand -------------------------
+ azure_parser = subparsers.add_parser('azure')
+ azure_parser.add_argument('--azure_vpc_name', type=str, help='Azure VPC Name')
+ azure_parser.add_argument('--azure_subnet_name', type=str, help='Azure Subnet Name')
+ azure_parser.add_argument('--azure_security_group_name', type=str, help='One Security group name for SSN')
+ azure_parser.add_argument('--azure_edge_security_group_name', type=str,
+ help='One Security group name for Edge node')
+ azure_parser.add_argument('--azure_resource_group_name', type=str, help='Name of Resource group in Azure')
+ azure_parser.add_argument('--azure_datalake_enable', type=str, default='false', choices=BOOL_CHOICES_LIST,
+ help='Provision DataLake storage account. (valid choices: %s)' % BOOL_CHOICES_LIST)
+ azure_parser.add_argument('--azure_ad_group_id', type=str, help='ID of Azure AD group')
+ azure_parser.add_argument('--azure_offer_number', type=str, help='Azure offer number')
+ azure_parser.add_argument('--azure_currency', type=str, help='Azure currency code')
+ azure_parser.add_argument('--azure_locale', type=str, help='Azure locale')
+ azure_parser.add_argument('--azure_application_id', type=str, help='Azure login application ID')
+ azure_parser.add_argument('--azure_validate_permission_scope', type=str, default='true',
+ choices=BOOL_CHOICES_LIST,
+ help='Azure permission scope validation. (valid choices: %s)' % BOOL_CHOICES_LIST)
+ azure_parser.add_argument('--azure_oauth2_enabled', type=str, default='false', choices=BOOL_CHOICES_LIST,
+ help='Using OAuth2 for logging in DataLab. (valid choices: %s)' % BOOL_CHOICES_LIST)
+ azure_parser.add_argument('--azure_region_info', type=str, help='Azure region info')
+ azure_parser.add_argument('--azure_source_vpc_name', type=str, help='Azure VPC source Name')
+ azure_parser.add_argument('--azure_source_resource_group_name', type=str, help='Azure source resource group')
+
+ azure_required_args = azure_parser.add_argument_group('Required arguments')
+ azure_required_args.add_argument('--azure_region', type=str, required=True, help='Azure region')
+ azure_required_args.add_argument('--azure_ssn_instance_size', type=str, default='Standard_DS2_v2', required=True,
+ help='The SSN instance shape')
+ azure_required_args.add_argument('--azure_auth_path', type=str, required=True,
+ help='Full path to Azure credentials JSON file')
+
+ # --------gcp subcommand -----------------------------
+ gcp_parser = subparsers.add_parser('gcp')
+ gcp_parser.add_argument('--billing_dataset_name', type=str,
+ help='Name of GCP dataset (BigQuery service) for billing')
+ gcp_parser.add_argument('--gcp_subnet_name', type=str, help='GCP Subnet Name')
+ gcp_parser.add_argument('--gcp_vpc_name', type=str, help='GCP VPC Name')
+ gcp_parser.add_argument('--gcp_firewall_name', type=str,
+ help='One of more comma-separated GCP Firewall rules for SSN')
+
+ gcp_required_args = gcp_parser.add_argument_group('Required arguments')
+ gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region')
+ gcp_required_args.add_argument('--gcp_zone', type=str, required=True, help='GCP zone')
+ gcp_required_args.add_argument('--gcp_ssn_instance_size', type=str, required=True, default='n1-standard-2',
+ help='The SSN instance shape')
+ gcp_required_args.add_argument('--gcp_project_id', type=str, required=True,
+ help='The project ID in Google Cloud Platform')
+ gcp_required_args.add_argument('--gcp_service_account_path', type=str, required=True,
+ help='The project ID in Google Cloud Platform')
+ return parser
def generate_docker_command():
diff --cc infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
index 04fa905,1263904..37e47e5
--- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
@@@ -29,8 -29,114 +29,116 @@@ import o
import time
import subprocess
import datalab.fab
+from datalab.logger import logging
+
+ def handle_dpkg_lock(error, rerun=False):
+ try:
+ count = 0
+ if 'E: Could not get lock ' and 'It is held by process ' in err:
+ log = datalab.fab.conn.sudo('cat /tmp/dpkg.log | grep "E: Could not get lock"').stdout
+ lock_path = log.split('\n')[0][22:log.find('.')]
+ pid = log.split('\n')[0][log.find('It is held by process ') + 22:].split(' ')[0]
+ datalab.fab.conn.sudo('kill -9 {}'.format(pid))
+ datalab.fab.conn.sudo('rm -f {}'.format(lock_path))
+ while 'no_lock' not in error and count < 10:
+ pid = datalab.fab.conn.sudo('lsof /var/lib/dpkg/lock-frontend | grep dpkg | awk \'{print $2}\'').stdout.replace( '\n', '')
+ if pid != '':
+ datalab.fab.conn.sudo('kill -9 {}'.format(pid))
+ datalab.fab.conn.sudo('rm -f /var/lib/dpkg/lock-frontend')
+
+ pid = datalab.fab.conn.sudo('lsof /var/lib/dpkg/lock | grep dpkg | awk \'{print $2}\'').stdout.replace('\n', '')
+ if pid != '':
+ datalab.fab.conn.sudo('kill -9 {}'.format(pid))
+ datalab.fab.conn.sudo('rm -f /var/lib/dpkg/lock')
+
+ if rerun:
+ datalab.fab.conn.sudo('dpkg --configure -a 2>&1 | tee /tmp/tee.tmp; '
+ 'if ! grep -w -E "({0})" /tmp/tee.tmp; '
+ 'then echo "no_lock" > /tmp/dpkg.log; '
+ 'else cat /tmp/tee.tmp > /tmp/dpkg.log;fi; '
+ 'if ! grep -w -E "({1})" /tmp/tee.tmp; '
+ 'then echo "no_error" >> /tmp/dpkg.log; '
+ 'else cat /tmp/tee.tmp >> /tmp/dpkg.log;fi'.format(lock_parser,
+ error_parser))
+ error = datalab.fab.conn.sudo('cat /tmp/dpkg.log').stdout
+ else:
+ error = 'no_lock'
+ count = count + 1
+ if 'no_error' not in error:
+ raise Exception
+ except:
+ sys.exit(1)
+
+ def handle_apt_lock(error, rerun=False):
+ try:
+ count = 0
+ if 'E: Could not get lock ' and 'It is held by process ' in err:
+ log = datalab.fab.conn.sudo('cat /tmp/apt.log | grep "E: Could not get lock"').stdout
+ lock_path = log.split('\n')[0][22:log.find('.')]
+ pid = log.split('\n')[0][log.find('It is held by process ') + 22:].split(' ')[0]
+ datalab.fab.conn.sudo('kill -9 {}'.format(pid))
+ datalab.fab.conn.sudo('rm -f {}'.format(lock_path))
+ while 'no_lock' not in error and count < 10:
+ pid = datalab.fab.conn.sudo('lsof /var/lib/apt/lists/lock | grep apt | awk \'{print $2}\'').stdout.replace('\n', '')
+ if pid != '':
+ datalab.fab.conn.sudo('kill -9 {}'.format(pid))
+ datalab.fab.conn.sudo('rm -f /var/lib/apt/lists/lock')
+
+ if rerun:
+ datalab.fab.conn.sudo('apt update 2>&1 | tee /tmp/tee.tmp; '
+ 'if ! grep -w -E "({0})" /tmp/tee.tmp; '
+ 'then echo "no_lock" > /tmp/apt.log; '
+ 'else cat /tmp/tee.tmp > /tmp/apt.log;fi; '
+ 'if ! grep -w -E "({1})" /tmp/tee.tmp; '
+ 'then echo "no_error" >> /tmp/apt.log; '
+ 'else cat /tmp/tee.tmp >> /tmp/apt.log;fi'.format(lock_parser,
+ error_parser))
+ error = datalab.fab.conn.sudo('cat /tmp/apt.log').stdout
+ else:
+ error = 'no_lock'
+ count = count + 1
+ if 'no_error' not in error:
+ raise Exception
+ except:
+ sys.exit(1)
+
+ def handle_apt_get_lock(error, rerun=False):
+ try:
+ count = 0
+ if 'E: Could not get lock ' and 'It is held by process ' in err:
+ log = datalab.fab.conn.sudo('cat /tmp/apt.log | grep "E: Could not get lock"').stdout
+ lock_path = log.split('\n')[0][22:log.find('.')]
+ pid = log.split('\n')[0][log.find('It is held by process ') + 22:].split(' ')[0]
+ datalab.fab.conn.sudo('kill -9 {}'.format(pid))
+ datalab.fab.conn.sudo('rm -f {}'.format(lock_path))
+ while 'no_lock' not in error and count < 10:
+ datalab.fab.conn.sudo('lsof /var/lib/dpkg/lock')
+ datalab.fab.conn.sudo('lsof /var/lib/apt/lists/lock')
+ datalab.fab.conn.sudo('lsof /var/cache/apt/archives/lock')
+ datalab.fab.conn.sudo('rm -f /var/lib/apt/lists/lock')
+ datalab.fab.conn.sudo('rm -f /var/cache/apt/archives/lock')
+ datalab.fab.conn.sudo('rm -f /var/lib/dpkg/lock')
+
+ if rerun:
+ datalab.fab.conn.sudo('apt-get {0} {1} 2>&1 | tee /tmp/tee.tmp; '
+ 'if ! grep -w -E "({2})" /tmp/tee.tmp; '
+ 'then echo "no_lock" > /tmp/apt_get.log; '
+ 'else cat /tmp/tee.tmp > /tmp/apt_get.log;fi; '
+ 'if ! grep -w -E "({3})" /tmp/tee.tmp; '
+ 'then echo "no_error" >> /tmp/apt_get.log; '
+ 'else cat /tmp/tee.tmp >> /tmp/apt_get.log;fi'.format(command,
+ requisites,
+ lock_parser,
+ error_parser))
+ error = datalab.fab.conn.sudo('cat /tmp/apt_get.log').stdout
+ else:
+ error = 'no_lock'
+ count = count + 1
+ if 'no_error' not in error:
+ raise Exception
+ except:
+ sys.exit(1)
+
def manage_pkg(command, environment, requisites):
try:
allow = False
diff --cc infrastructure-provisioning/src/general/lib/os/fab.py
index 707bc60,435efd8..c14a867
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@@ -38,275 -38,6 +38,278 @@@ from fabric import
from patchwork.files import exists
from patchwork import files
+
+# general functions for all resources
+def init_datalab_connection(hostname, username, keyfile):
+ try:
+ global conn
+ attempt = 0
+ while attempt < 15:
+ logging.info('connection attempt {}'.format(attempt))
+ conn = Connection(host=hostname, user=username, connect_kwargs={'banner_timeout': 200,
+ 'key_filename': keyfile})
+ conn.config.run.echo = True
+ try:
- conn.run('ls')
++ conn.run('hostname')
+ conn.config.run.echo = True
+ return conn
+ except:
+ attempt += 1
+ time.sleep(10)
++ if attempt == 15:
++ logging.info('Unable to establish connection')
++ raise Exception
+ except Exception as err:
+ logging.error('Function init_datalab_connection error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_pip(requisites):
+ try:
+ if not exists(conn, '/home/{}/.ensure_dir/pip_path_added'.format(os.environ['conf_os_user'])):
+ conn.sudo('bash -l -c "echo PATH=$PATH:/usr/local/bin/:/opt/spark/bin/ >> /etc/profile"')
+ conn.sudo('bash -l -c "echo export PATH >> /etc/profile"')
+ conn.sudo('pip3 install -UI pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
+ conn.sudo('pip3 install -U setuptools=={}'.format(os.environ['notebook_setuptools_version']))
+ conn.sudo('pip3 install -UI {} --no-cache-dir'.format(requisites))
+ conn.sudo('touch /home/{}/.ensure_dir/pip_path_added'.format(os.environ['conf_os_user']))
+ except Exception as err:
+ logging.error('Function ensure_pip error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def id_generator(size=10, chars=string.digits + string.ascii_letters):
+ return ''.join(random.choice(chars) for _ in range(size))
+
+
+def replace_multi_symbols(string, symbol, symbol_cut=False):
+ try:
+ symbol_amount = 0
+ for i in range(len(string)):
+ if string[i] == symbol:
+ symbol_amount = symbol_amount + 1
+ while symbol_amount > 1:
+ string = string.replace(symbol + symbol, symbol)
+ symbol_amount = symbol_amount - 1
+ if symbol_cut and string[-1] == symbol:
+ string = string[:-1]
+ return string
+ except Exception as err:
+ logging.error('Function replace_multi_symbols error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def append_result(error, exception=''):
+ try:
+ ts = time.time()
+ st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
+ if exception:
+ error_message = "[Error-{}]: {}. Exception: {}".format(st, error, str(exception))
+ logging.error(error_message)
+ else:
+ error_message = "[Error-{}]: {}.".format(st, error)
+ logging.error(error_message)
+ with open('/root/result.json', 'a+') as f:
+ text = f.read()
+ if len(text) == 0:
+ res = '{"error": ""}'
+ with open('/root/result.json', 'w') as f:
+ f.write(res)
+ with open("/root/result.json") as f:
+ data = json.load(f)
+ data['error'] = data['error'] + error_message
+ with open("/root/result.json", 'w') as f:
+ json.dump(data, f)
+ logging.error(data)
+ except Exception as err:
+ logging.error('Function append_result error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def put_resource_status(resource, status, datalab_path, os_user, hostname):
+ try:
+ keyfile = os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem"
+ init_datalab_connection(hostname, os_user, keyfile)
+ conn.sudo(
+ 'python3 ' + datalab_path + 'tmp/resource_status.py --resource {} --status {}'.format(resource, status))
+ conn.close()
+ except Exception as err:
+ logging.error('Function put_resource_status error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_ciphers():
+ try:
+ conn.sudo(
+ '''bash -c "echo -e '\nKexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256' >> /etc/ssh/sshd_config"''')
+ conn.sudo(
+ '''bash -c "echo -e 'Ciphers aes256-gcm@openssh.com,aes128-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr' >> /etc/ssh/sshd_config"''')
+ conn.sudo(
+ '''bash -c "echo -e '\tKexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256' >> /etc/ssh/ssh_config"''')
+ conn.sudo(
+ '''bash -c "echo -e '\tCiphers aes256-gcm@openssh.com,aes128-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr' >> /etc/ssh/ssh_config"''')
+ try:
+ conn.sudo('service ssh reload')
+ except:
+ conn.sudo('service sshd reload')
+ except Exception as err:
+ logging.error('Function pensure_ciphers error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def manage_npm_pkg(command):
+ try:
+ npm_count = 0
+ installed = False
+ npm_registry = ['https://registry.npmjs.org/', 'https://registry.npmjs.com/']
+ while not installed:
+ if npm_count > 60:
+ logging.error("NPM registry is not available, please try later")
+ sys.exit(1)
+ else:
+ try:
+ if npm_count % 2 == 0:
+ conn.sudo('npm config set registry {}'.format(npm_registry[0]))
+ else:
+ conn.sudo('npm config set registry {}'.format(npm_registry[1]))
+ conn.sudo('{}'.format(command))
+ installed = True
+ except:
+ npm_count += 1
+ time.sleep(50)
+ except Exception as err:
+ logging.error('Function manage_npm_pkg error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def update_hosts_file(os_user):
+ try:
+ if not exists(conn, '/home/{}/.ensure_dir/hosts_file_updated'.format(os_user)):
+ conn.sudo('sed -i "s/^127.0.0.1 localhost/127.0.0.1 localhost localhost.localdomain/g" /etc/hosts')
+ conn.sudo('touch /home/{}/.ensure_dir/hosts_file_updated'.format(os_user))
+ except Exception as err:
+ logging.error('Function update_hosts_file error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def install_certbot(os_user):
+ try:
+ if not exists(datalab.fab.conn, '/home/{}/.ensure_dir/certbot_ensured'.format(os_user)):
+ datalab.fab.conn.sudo('snap install core')
+ datalab.fab.conn.sudo('snap refresh core')
+ datalab.fab.conn.sudo('snap install --classic certbot')
+ datalab.fab.conn.sudo('ln -s /snap/bin/certbot /usr/bin/certbot')
+ datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/certbot_ensured'.format(os_user))
+ except Exception as err:
+ logging.error('Installing Certbot error: ' + str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def run_certbot(domain_name, node, email=''):
+ try:
+ if node == 'ssn':
+ datalab.fab.conn.sudo('service nginx stop')
+ else:
+ datalab.fab.conn.sudo('service openresty stop')
+ if email != '':
+ datalab.fab.conn.sudo('certbot certonly --standalone -n -d {}.{} -m {} --agree-tos'.format(node,
+ domain_name,
+ email))
+ else:
+ datalab.fab.conn.sudo('certbot certonly --standalone -n -d {}.{} --register-unsafely-without-email '
+ '--agree-tos'.format(node, domain_name))
+ except Exception as err:
+ logging.error('Running Certbot error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def configure_nginx_LE(domain_name, node):
+ try:
+ server_name_line =' server_name {}.{};'.format(node, domain_name)
+ cert_path_line = ' ssl_certificate /etc/letsencrypt/live/{}.{}/fullchain.pem;'.format(node, domain_name)
+ cert_key_line = ' ssl_certificate_key /etc/letsencrypt/live/{}.{}/privkey.pem;'.format(node, domain_name)
+ #certbot_service = "ExecStart = /usr/bin/certbot -q renew --pre-hook 'service nginx stop' --post-hook 'service nginx start'"
+ #certbot_service_path = '/lib/systemd/system/certbot.service'
+ if node == 'ssn':
+ nginx_config_path = '/etc/nginx/conf.d/nginx_proxy.conf'
+ else:
+ nginx_config_path = '/usr/local/openresty/nginx/conf/conf.d/proxy.conf'
+ datalab.fab.conn.sudo('sed -i "s|.* server_name .*|{}|" {}'.format(server_name_line, nginx_config_path))
+ datalab.fab.conn.sudo('sed -i "s|.* ssl_certificate .*|{}|" {}'.format(cert_path_line, nginx_config_path))
+ datalab.fab.conn.sudo('sed -i "s|.* ssl_certificate_key .*|{}|" {}'.format(cert_key_line, nginx_config_path))
+ #datalab.fab.conn.sudo('sed -i "s|.*ExecStart.*|{}|" {}'.format(certbot_service, certbot_service_path))
+ if node == 'ssn':
+ datalab.fab.conn.sudo('systemctl restart nginx')
+ else:
+ datalab.fab.conn.sudo('systemctl restart openresty')
+ except Exception as err:
+ logging.error('Configuring Nginx Let’s Encrypt certs error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+#function for edge node only
+def configure_http_proxy_server(config):
+ try:
+ if not exists(datalab.fab.conn,'/tmp/http_proxy_ensured'):
+ manage_pkg('-y install', 'remote', 'squid')
+ template_file = config['template_file']
+ proxy_subnet = config['exploratory_subnet']
+ conn.put(template_file, '/tmp/squid.conf')
+ conn.sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
+ conn.sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
+ replace_string = ''
+ for cidr in config['vpc_cidrs']:
+ replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
+ conn.sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
+ replace_string = ''
+ for cidr in config['allowed_ip_cidr']:
+ replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr)
+ conn.sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
+ conn.sudo('systemctl restart squid')
+ fab.conn.sudo('touch /tmp/http_proxy_ensured')
+ except Exception as err:
+ logging.error('Fai to install and configure squid:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def configure_nftables(config):
+ try:
+ if not exists(datalab.fab.conn,'/tmp/nftables_ensured'):
+ manage_pkg('-y install', 'remote', 'nftables')
+ conn.sudo('systemctl enable nftables.service')
+ conn.sudo('systemctl start nftables')
+ conn.sudo('sysctl net.ipv4.ip_forward=1')
+ if os.environ['conf_cloud_provider'] == 'aws':
+ interface = 'eth0'
+ elif os.environ['conf_cloud_provider'] == 'gcp':
+ interface = 'ens4'
+ conn.sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' /etc/sysctl.conf')
+ conn.sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/nftables.conf'.format(config['edge_ip']))
+ conn.sudo('sed -i "s|INTERFACE|{}|g" /opt/datalab/templates/nftables.conf'.format(interface))
+ conn.sudo(
+ 'sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet']))
+ conn.sudo('cp /opt/datalab/templates/nftables.conf /etc/')
+ conn.sudo('systemctl restart nftables')
+ conn.sudo('touch /tmp/nftables_ensured')
+ except Exception as err:
+ logging.error('Failed to configure nftables:', (err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+# functions for all computation resources
def ensure_python_venv(python_venv_version):
try:
if not exists(conn, '/opt/python/python{}'.format(python_venv_version)):
@@@ -737,274 -791,7 +740,274 @@@ def install_inactivity_checker(os_user
conn.sudo('systemctl start inactive.timer')
conn.sudo('touch /home/{}/.ensure_dir/inactive_ensured'.format(os_user))
except Exception as err:
- print('Failed to setup inactivity check service!', str(err))
+ logging.error('Function install_inactivity_checker error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def get_spark_memory(creds=False, os_user='', hostname='', keyfile=''):
+ try:
+ if creds:
+ con = init_datalab_connection(hostname, os_user, keyfile)
+ mem = con.sudo('free -m | grep Mem | tr -s " " ":" | cut -f 2 -d ":"').stdout.replace('\n', '')
+ instance_memory = int(mem)
+ else:
+ mem = conn.sudo('free -m | grep Mem | tr -s " " ":" | cut -f 2 -d ":"').stdout.replace('\n', '')
+ instance_memory = int(mem)
+ spark_memory = round(instance_memory * 90 / 100)
+ return spark_memory
+ except Exception as err:
+ logging.error('Function install_inactivity_checker error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+# functions for dataengine/dataengine-service resources
+def ensure_dataengine_service_devtools():
+ try:
+ if not exists(conn, '/home/{}/dataengine-service-devtools-ensured'.format(os.environ['conf_os_user'])):
+ if os.environ['conf_cloud_provider'] in 'aws':
+ manage_pkg('-y install', 'remote', 'libcurl libcurl-devel')
+ elif (os.environ['conf_cloud_provider'] in 'gcp') and (
+ '-w-' in conn.sudo('hostname').stdout.replace('\n', '')):
+ # manage_pkg('-y build-dep', 'remote', 'libcurl4-gnutls-dev libxml2-dev')
+ manage_pkg('-y install', 'remote', 'libxml2-dev libcurl4-openssl-dev pkg-config')
+ conn.sudo('R -e "install.packages(\'devtools\', repos = \'cloud.r-project.org\')"')
+ if (os.environ['conf_cloud_provider'] in 'gcp') and (
+ "R_LIBS_SITE" not in conn.sudo('cat /opt/conda/miniconda3/lib/R/etc/Renviron').stdout):
+ conn.sudo(
+ '''bash -l -c 'echo "R_LIBS_SITE=${R_LIBS_SITE-'/usr/local/lib/R/site-library:/usr/lib/R/site-library:/usr/lib/R/library'}" >> /opt/conda/miniconda3/lib/R/etc/Renviron' ''')
+ conn.sudo('touch /home/{}/dataengine-service-devtools-ensured'.format(os.environ['conf_os_user']))
+ except Exception as err:
+ logging.error('Function ensure_dataengine_service_devtools error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def configure_data_engine_service_livy(hostname, os_user, keyfile):
+ try:
+ init_datalab_connection(hostname, os_user, keyfile)
+ if exists(conn, '/usr/local/lib/livy'):
+ conn.sudo('rm -r /usr/local/lib/livy')
- conn.sudo('wget -P /tmp/ --user={} --password={} '
- '{}/repository/packages/livy.tar.gz --no-check-certificate'
- .format(os.environ['conf_repository_user'],
- os.environ['conf_repository_pass'], os.environ['conf_repository_address']))
++ conn.sudo('wget -P /tmp/ https://nexus.develop.dlabanalytics.com/repository/packages-public/livy.tar.gz '
++ '--no-check-certificate')
+ conn.sudo('tar -xzvf /tmp/livy.tar.gz -C /usr/local/lib/')
+ conn.sudo('ln -s /usr/local/lib/incubator-livy /usr/local/lib/livy')
+ conn.put('/root/templates/dataengine-service_livy-env.sh', '/usr/local/lib/livy/conf/livy-env.sh')
+ conn.put('/root/templates/dataengine-service_livy.service', '/tmp/livy.service')
+ conn.sudo("sed -i 's|OS_USER|{}|' /tmp/livy.service".format(os_user))
+ conn.sudo('mv /tmp/livy.service /etc/systemd/system/livy.service')
+ conn.sudo('systemctl daemon-reload')
+ conn.sudo('systemctl enable livy.service')
+ conn.sudo('systemctl start livy.service')
+ conn.close()
+ except Exception as err:
+ logging.error('Function configure_data_engine_service_livy error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def remove_rstudio_dataengines_kernel(cluster_name, os_user):
+ try:
+ cluster_re = ['-{}"'.format(cluster_name),
+ '-{}-'.format(cluster_name),
+ '-{}/'.format(cluster_name)]
+ conn.get('/home/{}/.Rprofile'.format(os_user), 'Rprofile')
+ data = open('Rprofile').read()
+ conf = filter(None, data.split('\n'))
+ # Filter config from any math of cluster_name in line,
+ # separated by defined symbols to avoid partly matches
+ conf = [i for i in conf if not any(x in i for x in cluster_re)]
+ comment_all = lambda x: x if x.startswith('#master') else '#{}'.format(x)
+ uncomment = lambda x: x[1:] if not x.startswith('#master') else x
+ conf = [comment_all(i) for i in conf]
+ conf = [uncomment(i) for i in conf]
+ last_spark = max([conf.index(i) for i in conf if 'master=' in i] or [0])
+ active_cluster = conf[last_spark].split('"')[-2] if last_spark != 0 else None
+ conf = conf[:last_spark] + [conf[l][1:] for l in range(last_spark, len(conf)) if conf[l].startswith("#")] \
+ + [conf[l] for l in range(last_spark, len(conf)) if not conf[l].startswith('#')]
+ with open('.Rprofile', 'w') as f:
+ for line in conf:
+ f.write('{}\n'.format(line))
+ conn.put('.Rprofile', '/home/{}/.Rprofile'.format(os_user))
+ conn.get('/home/{}/.Renviron'.format(os_user), 'Renviron')
+ data = open('Renviron').read()
+ conf = filter(None, data.split('\n'))
+ comment_all = lambda x: x if x.startswith('#') else '#{}'.format(x)
+ conf = [comment_all(i) for i in conf]
+ # Filter config from any math of cluster_name in line,
+ # separated by defined symbols to avoid partly matches
+ conf = [i for i in conf if not any(x in i for x in cluster_re)]
+ if active_cluster:
+ activate_cluster = lambda x: x[1:] if active_cluster in x else x
+ conf = [activate_cluster(i) for i in conf]
+ else:
+ last_spark = max([conf.index(i) for i in conf if 'SPARK_HOME' in i])
+ conf = conf[:last_spark] + [conf[l][1:] for l in range(last_spark, len(conf)) if conf[l].startswith("#")]
+ with open('.Renviron', 'w') as f:
+ for line in conf:
+ f.write('{}\n'.format(line))
+ conn.put('.Renviron', '/home/{}/.Renviron'.format(os_user))
+ if len(conf) == 1:
+ conn.sudo('rm -f /home/{}/.ensure_dir/rstudio_dataengine_ensured'.format(os_user))
+ conn.sudo('rm -f /home/{}/.ensure_dir/rstudio_dataengine-service_ensured'.format(os_user))
+ conn.sudo('''R -e "source('/home/{}/.Rprofile')"'''.format(os_user))
+ except Exception as err:
+ logging.error('Function remove_rstudio_dataengines_kernel error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+#following function should be checked if it needed
+def configure_data_engine_service_pip(hostname, os_user, keyfile, emr=False):
+ try:
+ init_datalab_connection(hostname, os_user, keyfile)
+ # datalab.common_lib.manage_pkg('-y install', 'remote', 'python3-pip')
+ if not exists(conn, '/usr/bin/pip3') and conn.sudo("python3.9 -V 2>/dev/null | awk '{print $2}'").stdout:
+ conn.sudo('ln -s /usr/bin/pip-3.9 /usr/bin/pip3')
+ elif not exists(conn, '/usr/bin/pip3') and conn.sudo("python3.8 -V 2>/dev/null | awk '{print $2}'").stdout:
+ conn.sudo('ln -s /usr/bin/pip-3.8 /usr/bin/pip3')
+ elif not exists(conn, '/usr/bin/pip3') and conn.sudo("python3.7 -V 2>/dev/null | awk '{print $2}'").stdout:
+ conn.sudo('ln -s /usr/bin/pip-3.7 /usr/bin/pip3')
+ elif not exists(conn, '/usr/bin/pip3') and conn.sudo("python3.6 -V 2>/dev/null | awk '{print $2}'").stdout:
+ conn.sudo('ln -s /usr/bin/pip-3.6 /usr/bin/pip3')
+ elif not exists(conn, '/usr/bin/pip3') and conn.sudo("python3.5 -V 2>/dev/null | awk '{print $2}'").stdout:
+ conn.sudo('ln -s /usr/bin/pip-3.5 /usr/bin/pip3')
+ if emr:
+ conn.sudo('pip3 install -U pip=={}'.format(os.environ['conf_pip_version']))
+ conn.sudo('ln -s /usr/local/bin/pip3.7 /bin/pip3.7')
+ conn.sudo('''bash -c -l 'echo "export PATH=$PATH:/usr/local/bin" >> /etc/profile' ''')
+ conn.sudo('bash -c -l "source /etc/profile"')
+ conn.run('bash -c -l "source /etc/profile"')
+ conn.close()
+ except Exception as err:
+ logging.error('Function configure_data_engine_service_pip error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+## following function should be removed after configurion kernels connection via live for all computation resources:
+def dataengine_dir_prepare(cluster_dir):
+ try:
+ subprocess.run('mkdir -p ' + cluster_dir, shell=True, check=True)
+ except Exception as err:
+ logging.error('Function dataengine_dir_prepare error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def ensure_dataengine_tensorflow_jars(jars_dir):
+ subprocess.run('wget https://dl.bintray.com/spark-packages/maven/tapanalyticstoolkit/spark-tensorflow-connector/'
+ '1.0.0-s_2.11/spark-tensorflow-connector-1.0.0-s_2.11.jar'
+ ' -O {}spark-tensorflow-connector-1.0.0-s_2.11.jar'.format(jars_dir), shell=True, check=True)
+
+
+def prepare(dataengine_service_dir, yarn_dir):
+ try:
+ subprocess.run('mkdir -p ' + dataengine_service_dir, shell=True, check=True)
+ subprocess.run('mkdir -p ' + yarn_dir, shell=True, check=True)
+ subprocess.run('sudo mkdir -p /opt/python/', shell=True, check=True)
+ result = os.path.exists(dataengine_service_dir + 'usr/')
+ return result
+ except Exception as err:
+ logging.error('Function prepare error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def configuring_notebook(dataengine_service_version):
+ try:
+ jars_path = '/opt/' + dataengine_service_version + '/jars/'
+ subprocess.run("""sudo bash -c "find """ + jars_path + """ -name '*netty*' | xargs rm -f" """, shell=True,
+ check=True)
+ except Exception as err:
+ logging.error('Function configuring_notebook error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+def find_cluster_kernels():
+ try:
+ de = [i for i in conn.sudo(
+ '''bash -l -c 'find /opt/ -maxdepth 1 -name "*-de-*" -type d | rev | cut -f 1 -d "/" | rev | xargs -r' ''').stdout.replace(
+ '\n', '').split(' ') if i != '']
+ des = [i for i in conn.sudo(
+ '''bash -l -c 'find /opt/ -maxdepth 2 -name "*-des-*" -type d | rev | cut -f 1,2 -d "/" | rev | xargs -r' ''').stdout.replace(
+ '\n', '').split(' ') if i != '']
+ return (de, des)
+ except Exception as err:
+ logging.error('Function find_cluster_kernels error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
+# functions for jupyter deeplearning and tensor notebooks
+def configure_jupyter(os_user, jupyter_conf_file, templates_dir, jupyter_version, exploratory_name):
+ if not exists(conn, '/home/' + os_user + '/.ensure_dir/jupyter_ensured'):
+ try:
+ if os.environ['conf_deeplearning_cloud_ami'] == 'false' or os.environ['application'] != 'deeplearning':
+ conn.sudo('pip3 install notebook=={} --no-cache-dir'.format(jupyter_version))
+ conn.sudo('pip3 install jupyter --no-cache-dir')
+ conn.sudo('rm -rf {}'.format(jupyter_conf_file))
+ conn.run('jupyter notebook --generate-config --config {}'.format(jupyter_conf_file))
+ conn.run('mkdir -p ~/.jupyter/custom/')
+ conn.run('echo "#notebook-container { width: auto; }" > ~/.jupyter/custom/custom.css')
+ conn.sudo('echo "c.NotebookApp.ip = \'0.0.0.0\'" >> {}'.format(jupyter_conf_file))
+ conn.sudo('echo "c.NotebookApp.base_url = \'/{0}/\'" >> {1}'.format(exploratory_name, jupyter_conf_file))
+ conn.sudo('echo c.NotebookApp.open_browser = False >> {}'.format(jupyter_conf_file))
+ conn.sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
+ conn.sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
+ conn.sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
+ if os.environ['conf_deeplearning_cloud_ami'] == 'true' and os.environ['application'] == 'deeplearning':
+ conn.sudo(
+ '''echo "c.NotebookApp.kernel_spec_manager_class = 'environment_kernels.EnvironmentKernelSpecManager'" >> {}'''.format(
+ jupyter_conf_file))
+ conn.sudo(
+ '''echo "c.EnvironmentKernelSpecManager.conda_env_dirs=['/home/ubuntu/anaconda3/envs']" >> {}'''.format(
+ jupyter_conf_file))
+ conn.put(templates_dir + 'jupyter-notebook.service', '/tmp/jupyter-notebook.service')
+ conn.sudo("chmod 644 /tmp/jupyter-notebook.service")
+ if os.environ['application'] == 'tensor':
+ conn.sudo(
+ "sed -i '/ExecStart/s|-c \"|-c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; |g' /tmp/jupyter-notebook.service")
+ elif os.environ['application'] == 'deeplearning' and os.environ['conf_deeplearning_cloud_ami'] == 'false':
+ conn.sudo(
+ "sed -i '/ExecStart/s|-c \"|-c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64:/usr/lib64/openmpi/lib: ; export PYTHONPATH=/home/" + os_user +
+ "/caffe/python:/home/" + os_user + "/pytorch/build:$PYTHONPATH ; |g' /tmp/jupyter-notebook.service")
+ conn.sudo("sed -i 's|CONF_PATH|{}|' /tmp/jupyter-notebook.service".format(jupyter_conf_file))
+ conn.sudo("sed -i 's|OS_USR|{}|' /tmp/jupyter-notebook.service".format(os_user))
- java_home = conn.run(
- "update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
++ if os.environ['application'] == 'deeplearning' and os.environ['conf_cloud_provider'] == 'azure':
++ java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-11.*/bin/java\'").stdout.splitlines()[0]
++ else:
++ java_home = conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
+ conn.sudo('sed -i \'/\[Service\]/ a\Environment=\"JAVA_HOME={}\"\' /tmp/jupyter-notebook.service'.format(
+ java_home))
+ conn.sudo('\cp /tmp/jupyter-notebook.service /etc/systemd/system/jupyter-notebook.service')
+ conn.sudo('chown -R {0}:{0} /home/{0}/.local'.format(os_user))
+ conn.sudo('mkdir -p /mnt/var')
+ conn.sudo('chown {0}:{0} /mnt/var'.format(os_user))
+ if os.environ['application'] == 'jupyter' or os.environ['application'] == 'deeplearning':
+ try:
+ conn.sudo('jupyter-kernelspec remove -f python3 || echo "Such kernel doesnt exists"')
+ conn.sudo('jupyter-kernelspec remove -f python2 || echo "Such kernel doesnt exists"')
+ except Exception as err:
+ logging.error('Error:', str(err))
+ conn.sudo("systemctl daemon-reload")
+ conn.sudo("systemctl enable jupyter-notebook")
+ conn.sudo("systemctl start jupyter-notebook")
+ conn.sudo('touch /home/{}/.ensure_dir/jupyter_ensured'.format(os_user))
+ except Exception as err:
+ logging.error('Function configure_jupyter error:', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+ else:
+ try:
+ conn.sudo(
+ 'sed -i "s/c.NotebookApp.base_url =.*/c.NotebookApp.base_url = \'\/{0}\/\'/" {1}'.format(
+ exploratory_name, jupyter_conf_file))
+ conn.sudo("systemctl restart jupyter-notebook")
+ except Exception as err:
+ logging.error('Function configure_jupyter error:', str(err))
+ traceback.print_exc()
sys.exit(1)
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
index c8135a2,94489da..7c0901f
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
@@@ -47,13 -47,13 +48,13 @@@ if __name__ == "__main__"
if role_name == '':
tag = {"Key": args.infra_tag_name, "Value": args.infra_tag_value}
user_tag = {"Key": "user:tag", "Value": args.user_tag_value}
- print("Creating role {0}, profile name {1}".format(args.role_name, args.role_profile_name))
+ logging.info("Creating role {0}, profile name {1}".format(args.role_name, args.role_profile_name))
- create_iam_role(args.role_name, args.role_profile_name, args.region, tag=tag, user_tag=user_tag)
+ create_iam_role(args.role_name, args.role_profile_name, args.region, args.permissions_boundary_arn, tag=tag, user_tag=user_tag)
else:
- print("ROLE AND ROLE PROFILE ARE ALREADY CREATED")
- print("ROLE {} created. IAM group {} created".format(args.role_name, args.role_profile_name))
+ logging.info("ROLE AND ROLE PROFILE ARE ALREADY CREATED")
+ logging.info("ROLE {} created. IAM group {} created".format(args.role_name, args.role_profile_name))
- print("ATTACHING POLICIES TO ROLE")
+ logging.info("ATTACHING POLICIES TO ROLE")
if args.policy_file_name != '':
create_attach_policy(args.policy_name, args.role_name, args.policy_file_name)
else:
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
index 6a22967,43261d6..06a873e
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
@@@ -31,23 -31,60 +31,24 @@@ import o
import sys
import traceback
import subprocess
+ import uuid
from fabric import *
-if __name__ == "__main__":
- local_log_filename = "{}_{}.log".format(os.environ['conf_resource'], os.environ['request_id'])
- local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
- logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
- level=logging.DEBUG,
- filename=local_log_filepath)
-
- ssn_conf = dict()
- ssn_conf['instance'] = 'ssn'
-
- def clear_resources():
- if ssn_conf['domain_created']:
- datalab.actions_lib.remove_route_53_record(os.environ['ssn_hosted_zone_id'],
- os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'])
- datalab.actions_lib.remove_ec2(ssn_conf['tag_name'], ssn_conf['instance_name'])
- datalab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
- datalab.actions_lib.remove_s3(ssn_conf['instance'])
- if ssn_conf['pre_defined_sg']:
- datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
- if ssn_conf['pre_defined_subnet']:
- datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
- if ssn_conf['pre_defined_vpc']:
- datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
- datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
- if ssn_conf['pre_defined_vpc2']:
- datalab.actions_lib.remove_peering('*')
- try:
- datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
- except:
- print("There are no VPC Endpoints")
- datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+def cleanup_aws_resources(tag_name, service_base_name):
+ try:
+ params = "--tag_name {} --service_base_name {}".format(tag_name, service_base_name)
+ subprocess.run("~/scripts/{}.py {}".format('ssn_terminate_aws_resources', params), shell=True, check=True)
+ except:
+ traceback.print_exc()
+ raise Exception
+if __name__ == "__main__":
+ # deriving variables for ssn node deployment
try:
logging.info('[DERIVING NAMES]')
- print('[DERIVING NAMES]')
+ ssn_conf = dict()
ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] = datalab.fab.replace_multi_symbols(
- os.environ['conf_service_base_name'][:20], '-', True)
- if 'ssn_hosted_zone_id' in os.environ and 'ssn_hosted_zone_name' in os.environ and \
- 'ssn_subdomain' in os.environ:
- ssn_conf['domain_created'] = True
- else:
- ssn_conf['domain_created'] = False
- ssn_conf['pre_defined_vpc'] = False
- ssn_conf['pre_defined_subnet'] = False
- ssn_conf['pre_defined_sg'] = False
- ssn_conf['billing_enabled'] = True
+ os.environ['conf_service_base_name'][:20], '-', True)
ssn_conf['role_name'] = '{}-ssn-role'.format(ssn_conf['service_base_name'])
ssn_conf['role_profile_name'] = '{}-ssn-profile'.format(ssn_conf['service_base_name'])
ssn_conf['policy_name'] = '{}-ssn-policy'.format(ssn_conf['service_base_name'])
@@@ -62,67 -98,96 +63,71 @@@
ssn_conf['sg_name'] = '{}-ssn-sg'.format(ssn_conf['service_base_name'])
ssn_conf['network_type'] = os.environ['conf_network_type']
ssn_conf['datalab_ssh_user'] = os.environ['conf_os_user']
-
- try:
- if os.environ['aws_vpc_id'] == '':
- raise KeyError
- except KeyError:
- ssn_conf['tag'] = {"Key": ssn_conf['tag_name'], "Value": "{}-subnet".format(ssn_conf['service_base_name'])}
- os.environ['aws_vpc_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- ssn_conf['pre_defined_vpc'] = True
- try:
- if os.environ['aws_subnet_id'] == '':
- raise KeyError
- except KeyError:
- ssn_conf['tag'] = {"Key": ssn_conf['tag_name'], "Value": "{}-subnet".format(ssn_conf['service_base_name'])}
- os.environ['aws_subnet_id'] = datalab.meta_lib.get_subnet_by_tag(ssn_conf['tag'], True)
- ssn_conf['pre_defined_subnet'] = True
- try:
- if os.environ['conf_duo_vpc_enable'] == 'true' and not os.environ['aws_vpc2_id']:
- raise KeyError
- except KeyError:
- ssn_conf['tag'] = {"Key": ssn_conf['tag2_name'], "Value": "{}-subnet".format(ssn_conf['service_base_name'])}
- os.environ['aws_vpc2_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
- ssn_conf['service_base_name'])
- ssn_conf['pre_defined_vpc2'] = True
- try:
- if os.environ['conf_duo_vpc_enable'] == 'true' and not os.environ['aws_peering_id']:
- raise KeyError
- except KeyError:
- os.environ['aws_peering_id'] = datalab.meta_lib.get_peering_by_tag(ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- ssn_conf['pre_defined_peering'] = True
- try:
- if os.environ['aws_security_groups_ids'] == '':
- raise KeyError
- except KeyError:
- os.environ['aws_security_groups_ids'] = datalab.meta_lib.get_security_group_by_name(ssn_conf['sg_name'])
- ssn_conf['pre_defined_sg'] = True
- try:
- if os.environ['aws_account_id'] == '':
- raise KeyError
- if os.environ['aws_billing_bucket'] == '':
- raise KeyError
- except KeyError:
+ ssn_conf['ssn_datalab_path'] = os.environ['ssn_datalab_path']
+ ssn_conf['conf_tag_resource_id'] = os.environ['conf_tag_resource_id']
+ ssn_conf['instance_hostname'] = (lambda x: datalab.meta_lib.get_instance_ip_address(
+ ssn_conf['tag_name'], ssn_conf['instance_name']).get(
+ 'Private') if x == 'private' else datalab.meta_lib.get_instance_hostname(
+ ssn_conf['tag_name'], ssn_conf['instance_name']))(ssn_conf['network_type'])
+ ssn_conf['initial_user'] = (lambda x: 'ubuntu' if x == 'debian' else 'ec2-user')(os.environ['conf_os_family'])
+ ssn_conf['sudo_group'] = (lambda x: 'sudo' if x == 'debian' else 'wheel')(os.environ['conf_os_family'])
+ ssn_conf['step_cert_sans'] = (lambda x: (lambda x: ' --san {0} --san {1}'.format(
+ datalab.meta_lib.get_instance_hostname(ssn_conf['tag_name'], ssn_conf['instance_name']),
+ datalab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'], ssn_conf['instance_name']).get(
+ 'Public')) if x == 'public' else ' --san {0}'.format(datalab.meta_lib.get_instance_ip_address(
+ ssn_conf['tag_name'], ssn_conf['instance_name']).get('Private')))(
+ ssn_conf['network_type']) if x == 'true' else '')(os.environ['conf_stepcerts_enabled'])
+ if 'aws_vpc_id' in os.environ and os.environ['aws_vpc_id'] != '':
+ ssn_conf['aws_vpc_id'] = os.environ['aws_vpc_id']
+ else:
+ ssn_conf['aws_vpc_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'],
+ ssn_conf['service_base_name'])
+ if os.environ['conf_duo_vpc_enable'] == 'true' and 'aws_vpc2_id' in os.environ\
+ and os.environ['aws_vpc2_id'] != '':
+ ssn_conf['aws_vpc2_id'] = os.environ['aws_vpc2_id']
+ else:
+ ssn_conf['aws_vpc2_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
+ ssn_conf['service_base_name'])
+ if os.environ['conf_duo_vpc_enable'] == 'true' and not os.environ['aws_peering_id']:
+ ssn_conf['aws_peering_id'] = datalab.meta_lib.get_peering_by_tag(ssn_conf['tag_name'],
+ ssn_conf['service_base_name'])
+ elif os.environ['conf_duo_vpc_enable'] == 'true' and aws_peering_id in os.environ \
+ and os.environ['aws_peering_id'] != '':
+ ssn_conf['aws_peering_id'] = os.environ['aws_peering_id']
+ else:
+ ssn_conf['aws_peering_id'] = None
+ if 'aws_subnet_id' in os.environ and os.environ['aws_subnet_id'] != '':
+ ssn_conf['aws_subnet_id'] = os.environ['aws_subnet_ids']
+ else:
+ ssn_conf['aws_subnet_id'] = datalab.meta_lib.get_subnet_by_tag(ssn_conf['subnet_tag'], True)
+ if 'aws_security_groups_ids' in os.environ and os.environ['aws_security_groups_ids'] != '':
+ ssn_conf['aws_security_groups_ids'] = os.environ['aws_security_groups_ids']
+ else:
+ ssn_conf['aws_security_groups_ids'] = datalab.meta_lib.get_security_group_by_name(ssn_conf['sg_name'])
+ if 'aws_billing_bucket' in os.environ and os.environ['aws_billing_bucket'] == '':
+ ssn_conf['billing_enabled'] = True
+ ssn_conf['aws_billing_bucket'] = os.environ['aws_billing_bucket']
+ else:
ssn_conf['billing_enabled'] = False
- if not ssn_conf['billing_enabled']:
- os.environ['aws_account_id'] = 'None'
- os.environ['aws_billing_bucket'] = 'None'
- try:
- if not os.environ['aws_report_path']:
- raise KeyError
- except KeyError:
- os.environ['aws_report_path'] = ''
-
+ ssn_conf['aws_billing_bucket'] = 'None'
+ if 'aws_report_path' in os.environ and os.environ['aws_report_path'] == '':
+ ssn_conf['aws_report_path'] = os.environ['aws_report_path']
+ else:
+ ssn_conf['aws_report_path'] = ''
+ if 'keycloak_client_name' not in os.environ:
+ os.environ['keycloak_client_name'] = '{}-ui'.format(ssn_conf['service_base_name'])
+ if 'keycloak_client_secret' not in os.environ:
+ os.environ['keycloak_client_secret'] = str(uuid.uuid4())
except Exception as err:
+ logging.error('Error: {0}'.format(err))
datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
- clear_resources()
+ cleanup_aws_resources(ssn_conf['tag_name'], ssn_conf['service_base_name'])
+ traceback.print_exc()
sys.exit(1)
+ #creating datalab ssh user
try:
- if os.environ['conf_os_family'] == 'debian':
- ssn_conf['initial_user'] = 'ubuntu'
- ssn_conf['sudo_group'] = 'sudo'
- if os.environ['conf_os_family'] == 'redhat':
- ssn_conf['initial_user'] = 'ec2-user'
- ssn_conf['sudo_group'] = 'wheel'
-
- if ssn_conf['network_type'] == 'private':
- ssn_conf['instance_hostname'] = datalab.meta_lib.get_instance_ip_address(
- ssn_conf['tag_name'], ssn_conf['instance_name']).get('Private')
- else:
- ssn_conf['instance_hostname'] = datalab.meta_lib.get_instance_hostname(
- ssn_conf['tag_name'], ssn_conf['instance_name'])
-
- if os.environ['conf_stepcerts_enabled'] == 'true':
- ssn_conf['step_cert_sans'] = ' --san {0} '.format(datalab.meta_lib.get_instance_ip_address(
- ssn_conf['tag_name'], ssn_conf['instance_name']).get('Private'))
- if ssn_conf['network_type'] == 'public':
- ssn_conf['step_cert_sans'] += ' --san {0} --san {1}'.format(
- datalab.meta_lib.get_instance_hostname(ssn_conf['tag_name'], ssn_conf['instance_name']),
- datalab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
- ssn_conf['instance_name']).get('Public'))
- else:
- ssn_conf['step_cert_sans'] = ''
-
logging.info('[CREATING DATALAB SSH USER]')
- print('[CREATING DATALAB SSH USER]')
params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format(
ssn_conf['instance_hostname'], os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem",
ssn_conf['initial_user'], ssn_conf['datalab_ssh_user'], ssn_conf['sudo_group'])
@@@ -213,23 -274,11 +218,30 @@@
traceback.print_exc()
raise Exception
except Exception as err:
- datalab.fab.append_result("Unable to configure docker.", str(err))
- clear_resources()
+ logging.error('Error: {0}'.format(err))
+ datalab.fab.append_result("Failed to configure docker.", str(err))
+ cleanup_aws_resources(ssn_conf['tag_name'], ssn_conf['service_base_name'])
+ sys.exit(1)
+
+ #configuring keycloak client for ui
+ try:
+ logging.info('[CONFIGURE KEYCLOAK CLIENT FOR DATALAB UI]')
++ keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
++ "--keycloak_user {} --keycloak_user_password {} --instance_public_ip {} --keycloak_client_secret {} " \
++ .format(ssn_conf['service_base_name'], os.environ['keycloak_auth_server_url'],
++ os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
++ os.environ['keycloak_user_password'], datalab.meta_lib.get_instance_hostname(
++ ssn_conf['tag_name'], ssn_conf['instance_name']), os.environ['keycloak_client_secret'])
++ subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
+ except Exception as err:
+ logging.error('Error: {0}'.format(err))
+ datalab.fab.append_result("Failed to configure Keycloak client for DataLab UI.", str(err))
+ cleanup_aws_resources(ssn_conf['tag_name'], ssn_conf['service_base_name'])
sys.exit(1)
+ #configuring UI
try:
+ logging.info('[CONFIGURE SSN INSTANCE UI]')
cloud_params = [
{
'key': 'KEYCLOAK_REDIRECT_URI',
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index 61bcf72,1482297..7e21cb1
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@@ -248,21 -279,36 +248,23 @@@ if __name__ == "__main__"
except:
traceback.print_exc()
raise Exception
- with open('/tmp/ssn_sg_id', 'r') as f:
- os.environ['aws_security_groups_ids'] = f.read()
- except Exception as err:
- datalab.gab_lib.append_result("Failed creating security group for SSN.", str(err))
- if ssn_conf['pre_defined_vpc']:
- datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
- datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
- if ssn_conf['pre_defined_vpc2']:
- datalab.actions_lib.remove_peering('*')
- try:
- datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
- except:
- print("There are no VPC Endpoints")
- datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
- sys.exit(1)
+ ssn_conf['aws_security_groups_ids'] = datalab.meta_lib.get_security_group_by_name(ssn_conf['sg_name'])
+ except Exception as err:
+ logging.error('Error: {0}'.format(err))
+ datalab.fab.append_result("Failed to create security group for SSN", str(err))
+ cleanup_aws_resources(ssn_conf['tag_name'], ssn_conf['service_base_name'])
+ sys.exit(1)
+ #creating roles
try:
logging.info('[CREATE ROLES]')
- print('[CREATE ROLES]')
params = "--role_name {} --role_profile_name {} --policy_name {} --policy_file_name {} --region {} " \
- "--infra_tag_name {} --infra_tag_value {} --user_tag_value {}".\
+ "--infra_tag_name {} --infra_tag_value {} --user_tag_value {}". \
format(ssn_conf['role_name'], ssn_conf['role_profile_name'], ssn_conf['policy_name'],
- ssn_conf['policy_path'], os.environ['aws_region'], ssn_conf['tag_name'],
+ ssn_conf['policy_path'], ssn_conf['region'], ssn_conf['tag_name'],
ssn_conf['service_base_name'], ssn_conf['user_tag'])
+ if 'aws_permissions_boundary_arn' in os.environ:
+ params = '{} --permissions_boundary_arn {}'.format(params, os.environ['aws_permissions_boundary_arn'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_role_policy', params), shell=True, check=True)
except:
diff --cc infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index 8c0e4cb,3a1f062..1dd8673
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@@ -111,23 -116,39 +111,37 @@@ def build_docker_images(image_list, reg
host_string = '{}@{}'.format(args.os_user, args.hostname)
if os.environ['conf_cloud_provider'] == 'azure':
conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
- 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
+ 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
- '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
- if region == 'cn-north-1':
- add_china_repository(datalab_path)
+ '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
- for image in image_list:
- name = image['name']
- tag = image['tag']
- conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json '
+ if 'conf_repository_user' in os.environ and 'conf_repository_port' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ and os.environ['conf_download_docker_images'] == 'true':
+ conn.sudo('sudo docker login -u {0} -p {1} {2}:{3}'
+ .format(os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address'], os.environ['conf_repository_port']))
+ for image in image_list:
+ name = image['name']
+ tag = image['tag']
+ conn.sudo('docker pull {0}:{4}/docker.datalab-{2}-{1}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
+ conn.sudo('docker image tag {0}:{4}/docker.datalab-{2}-{1}:{3} docker.datalab-{2}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
+ conn.sudo('docker image rm {0}:{4}/docker.datalab-{2}-{1}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
+ return True
+ else:
+ for image in image_list:
+ name = image['name']
+ tag = image['tag']
+ conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json '
- '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name))
+ '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name))
- if name == 'base':
- conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
+ if name == 'base':
+ conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
- "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} "
- ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
+ "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} "
+ ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
- else:
- conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
+ else:
+ conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
- "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
+ "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
- conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path))
- return True
+ conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path))
+ return True
except:
return False
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org