You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by of...@apache.org on 2020/09/30 11:36:18 UTC
[incubator-datalab] 01/01: Merge branch 'new-datalab-name' into
develop
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit b19b207b070e8edb9e3a0a5c2fb61185e4e41d89
Merge: 7598662 66c13a3
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Wed Sep 30 14:35:46 2020 +0300
Merge branch 'new-datalab-name' into develop
CONTRIBUTING.md | 6 +-
README.md | 380 +++---
RELEASE_NOTES.md | 16 +-
USER_GUIDE.md | 58 +-
build.properties | 2 +-
doc/{dlab_aws.png => datalab_aws.png} | Bin
doc/{dlab_azure.png => datalab_azure.png} | Bin
doc/{dlab_gcp.png => datalab_gcp.png} | Bin
.../scripts/POST_DEPLOYMENT.md | 10 +-
.../scripts/deploy_datalab.py | 231 ++++
infrastructure-provisioning/scripts/deploy_dlab.py | 220 ----
.../scripts/deploy_repository/deploy_repository.py | 36 +-
.../scripts/post-deployment_configuration.py | 162 ++-
.../src/base/scripts/install_prerequisites.py | 13 +-
.../src/base/scripts/install_user_key.py | 6 +-
.../src/dataengine-service/fabfile.py | 19 +-
.../src/dataengine/fabfile.py | 20 +-
.../src/dataengine/scripts/configure_dataengine.py | 10 +-
.../src/deeplearning/fabfile.py | 12 +-
.../scripts/configure_deep_learning_node.py | 13 +-
infrastructure-provisioning/src/edge/fabfile.py | 10 +-
.../src/edge/scripts/configure_http_proxy.py | 5 +-
.../edge/scripts/configure_nginx_reverse_proxy.py | 8 +-
.../src/edge/scripts/reupload_ssh_key.py | 9 +-
.../src/edge/templates/squid.conf | 4 +-
.../src/general/api/check_inactivity.py | 9 +-
.../src/general/api/configure.py | 9 +-
.../src/general/api/create.py | 9 +-
.../src/general/api/create_image.py | 9 +-
.../src/general/api/git_creds.py | 9 +-
.../src/general/api/install_libs.py | 9 +-
.../src/general/api/list_libs.py | 16 +-
.../src/general/api/reconfigure_spark.py | 9 +-
.../src/general/api/recreate.py | 9 +-
.../src/general/api/reupload_key.py | 11 +-
.../src/general/api/start.py | 9 +-
.../src/general/api/status.py | 9 +-
.../src/general/api/stop.py | 9 +-
.../src/general/api/terminate.py | 9 +-
.../src/general/api/terminate_image.py | 9 +-
.../src/general/conf/datalab.ini | 377 ++++++
.../src/general/conf/dlab.ini | 377 ------
.../src/general/files/aws/base_Dockerfile | 8 +-
.../src/general/files/aws/base_description.json | 2 +-
.../files/aws/dataengine-service_Dockerfile | 6 +-
.../src/general/files/aws/dataengine_Dockerfile | 4 +-
.../src/general/files/aws/deeplearning_Dockerfile | 4 +-
.../src/general/files/aws/edge_Dockerfile | 4 +-
.../src/general/files/aws/jupyter_Dockerfile | 4 +-
.../src/general/files/aws/jupyterlab_Dockerfile | 4 +-
.../src/general/files/aws/project_Dockerfile | 4 +-
.../src/general/files/aws/rstudio_Dockerfile | 4 +-
.../src/general/files/aws/ssn_Dockerfile | 4 +-
.../src/general/files/aws/ssn_description.json | 2 +-
.../general/files/aws/tensor-rstudio_Dockerfile | 4 +-
.../src/general/files/aws/tensor_Dockerfile | 4 +-
.../src/general/files/aws/zeppelin_Dockerfile | 4 +-
.../src/general/files/azure/base_Dockerfile | 8 +-
.../src/general/files/azure/base_description.json | 2 +-
.../src/general/files/azure/dataengine_Dockerfile | 4 +-
.../general/files/azure/deeplearning_Dockerfile | 4 +-
.../src/general/files/azure/edge_Dockerfile | 4 +-
.../src/general/files/azure/jupyter_Dockerfile | 4 +-
.../src/general/files/azure/jupyterlab_Dockerfile | 4 +-
.../src/general/files/azure/project_Dockerfile | 4 +-
.../src/general/files/azure/rstudio_Dockerfile | 4 +-
.../src/general/files/azure/ssn_Dockerfile | 4 +-
.../src/general/files/azure/ssn_description.json | 2 +-
.../src/general/files/azure/tensor_Dockerfile | 4 +-
.../src/general/files/azure/zeppelin_Dockerfile | 4 +-
.../src/general/files/gcp/base_Dockerfile | 8 +-
.../src/general/files/gcp/base_description.json | 2 +-
.../files/gcp/dataengine-service_Dockerfile | 4 +-
.../src/general/files/gcp/dataengine_Dockerfile | 4 +-
.../src/general/files/gcp/deeplearning_Dockerfile | 4 +-
.../src/general/files/gcp/edge_Dockerfile | 4 +-
.../src/general/files/gcp/jupyter_Dockerfile | 4 +-
.../src/general/files/gcp/jupyterlab_Dockerfile | 4 +-
.../src/general/files/gcp/project_Dockerfile | 4 +-
.../src/general/files/gcp/rstudio_Dockerfile | 4 +-
.../src/general/files/gcp/ssn_Dockerfile | 4 +-
.../src/general/files/gcp/ssn_description.json | 2 +-
.../src/general/files/gcp/superset_Dockerfile | 4 +-
.../general/files/gcp/tensor-rstudio_Dockerfile | 4 +-
.../src/general/files/gcp/tensor_Dockerfile | 4 +-
.../src/general/files/gcp/zeppelin_Dockerfile | 4 +-
.../src/general/lib/aws/actions_lib.py | 41 +-
.../src/general/lib/aws/meta_lib.py | 16 +-
.../src/general/lib/azure/actions_lib.py | 70 +-
.../src/general/lib/gcp/actions_lib.py | 46 +-
.../src/general/lib/gcp/meta_lib.py | 20 +-
.../src/general/lib/os/debian/edge_lib.py | 47 +-
.../src/general/lib/os/debian/notebook_lib.py | 21 +-
.../src/general/lib/os/debian/ssn_lib.py | 137 +-
.../src/general/lib/os/fab.py | 96 +-
.../src/general/lib/os/redhat/edge_lib.py | 43 +-
.../src/general/lib/os/redhat/notebook_lib.py | 20 +-
.../src/general/lib/os/redhat/ssn_lib.py | 128 +-
.../src/general/scripts/aws/common_collect_data.py | 15 +-
.../general/scripts/aws/common_create_bucket.py | 7 +-
.../general/scripts/aws/common_create_instance.py | 5 +-
.../scripts/aws/common_create_notebook_image.py | 24 +-
.../general/scripts/aws/common_create_policy.py | 7 +-
.../scripts/aws/common_create_role_policy.py | 4 +-
.../scripts/aws/common_create_security_group.py | 8 +-
.../general/scripts/aws/common_create_subnet.py | 29 +-
.../scripts/aws/common_download_git_certfile.py | 8 +-
...common_notebook_configure_dataengine-service.py | 62 +-
.../aws/common_notebook_configure_dataengine.py | 39 +-
.../general/scripts/aws/common_prepare_notebook.py | 50 +-
.../general/scripts/aws/common_put_to_bucket.py | 2 +-
.../scripts/aws/common_remove_remote_kernels.py | 7 +-
.../src/general/scripts/aws/common_reupload_key.py | 7 +-
.../general/scripts/aws/common_start_notebook.py | 33 +-
.../general/scripts/aws/common_stop_notebook.py | 41 +-
.../scripts/aws/common_terminate_notebook.py | 29 +-
.../scripts/aws/common_terminate_notebook_image.py | 15 +-
.../scripts/aws/dataengine-service_configure.py | 86 +-
.../scripts/aws/dataengine-service_create.py | 23 +-
.../scripts/aws/dataengine-service_install_libs.py | 12 +-
.../scripts/aws/dataengine-service_list_libs.py | 8 +-
.../scripts/aws/dataengine-service_prepare.py | 60 +-
.../scripts/aws/dataengine-service_terminate.py | 27 +-
.../general/scripts/aws/dataengine_configure.py | 123 +-
.../src/general/scripts/aws/dataengine_prepare.py | 81 +-
.../src/general/scripts/aws/dataengine_start.py | 22 +-
.../src/general/scripts/aws/dataengine_stop.py | 14 +-
.../general/scripts/aws/dataengine_terminate.py | 20 +-
.../general/scripts/aws/deeplearning_configure.py | 133 +-
.../scripts/aws/edge_associate_elastic_ip.py | 10 +-
.../src/general/scripts/aws/edge_configure.py | 109 +-
.../scripts/aws/edge_configure_route_table.py | 5 +-
.../src/general/scripts/aws/edge_start.py | 22 +-
.../src/general/scripts/aws/edge_status.py | 17 +-
.../src/general/scripts/aws/edge_stop.py | 19 +-
.../src/general/scripts/aws/jupyter_configure.py | 125 +-
.../jupyter_dataengine-service_create_configs.py | 17 +-
.../jupyter_install_dataengine-service_kernels.py | 31 +-
.../general/scripts/aws/jupyterlab_configure.py | 141 +-
.../src/general/scripts/aws/project_prepare.py | 179 ++-
.../src/general/scripts/aws/project_terminate.py | 66 +-
.../src/general/scripts/aws/rstudio_configure.py | 139 +-
.../rstudio_dataengine-service_create_configs.py | 16 +-
.../rstudio_install_dataengine-service_kernels.py | 19 +-
.../scripts/aws/ssn_associate_elastic_ip.py | 10 +-
.../src/general/scripts/aws/ssn_configure.py | 147 +--
.../src/general/scripts/aws/ssn_create_endpoint.py | 10 +-
.../src/general/scripts/aws/ssn_create_vpc.py | 5 +-
.../src/general/scripts/aws/ssn_finalize.py | 5 +-
.../src/general/scripts/aws/ssn_prepare.py | 296 +++--
.../src/general/scripts/aws/ssn_terminate.py | 23 +-
.../scripts/aws/ssn_terminate_aws_resources.py | 87 +-
.../scripts/aws/tensor-rstudio_configure.py | 143 +--
.../src/general/scripts/aws/tensor_configure.py | 129 +-
.../src/general/scripts/aws/zeppelin_configure.py | 133 +-
.../zeppelin_dataengine-service_create_configs.py | 18 +-
.../zeppelin_install_dataengine-service_kernels.py | 20 +-
.../general/scripts/azure/common_collect_data.py | 15 +-
.../azure/common_create_datalake_directory.py | 7 +-
.../scripts/azure/common_create_instance.py | 12 +-
.../scripts/azure/common_create_notebook_image.py | 30 +-
.../scripts/azure/common_create_security_group.py | 6 +-
.../scripts/azure/common_create_storage_account.py | 8 +-
.../general/scripts/azure/common_create_subnet.py | 15 +-
.../scripts/azure/common_download_git_certfile.py | 7 +-
.../azure/common_notebook_configure_dataengine.py | 31 +-
.../scripts/azure/common_prepare_notebook.py | 36 +-
.../scripts/azure/common_remove_remote_kernels.py | 7 +-
.../general/scripts/azure/common_reupload_key.py | 7 +-
.../general/scripts/azure/common_start_notebook.py | 27 +-
.../general/scripts/azure/common_stop_notebook.py | 23 +-
.../scripts/azure/common_terminate_notebook.py | 23 +-
.../azure/common_terminate_notebook_image.py | 13 +-
.../general/scripts/azure/dataengine_configure.py | 97 +-
.../general/scripts/azure/dataengine_prepare.py | 39 +-
.../src/general/scripts/azure/dataengine_start.py | 21 +-
.../src/general/scripts/azure/dataengine_stop.py | 21 +-
.../general/scripts/azure/dataengine_terminate.py | 23 +-
.../scripts/azure/deeplearning_configure.py | 91 +-
.../src/general/scripts/azure/edge_configure.py | 53 +-
.../src/general/scripts/azure/edge_prepare.py | 15 +-
.../src/general/scripts/azure/edge_start.py | 18 +-
.../src/general/scripts/azure/edge_status.py | 15 +-
.../src/general/scripts/azure/edge_stop.py | 20 +-
.../src/general/scripts/azure/edge_terminate.py | 35 +-
.../src/general/scripts/azure/jupyter_configure.py | 85 +-
.../general/scripts/azure/jupyterlab_configure.py | 99 +-
.../src/general/scripts/azure/project_prepare.py | 39 +-
.../src/general/scripts/azure/project_terminate.py | 35 +-
.../general/scripts/azure/rstudio_change_pass.py | 6 +-
.../src/general/scripts/azure/rstudio_configure.py | 95 +-
.../src/general/scripts/azure/ssn_configure.py | 76 +-
.../general/scripts/azure/ssn_create_datalake.py | 8 +-
.../general/scripts/azure/ssn_create_peering.py | 7 +-
.../scripts/azure/ssn_create_resource_group.py | 5 +-
.../src/general/scripts/azure/ssn_create_vpc.py | 5 +-
.../src/general/scripts/azure/ssn_prepare.py | 50 +-
.../src/general/scripts/azure/ssn_terminate.py | 50 +-
.../src/general/scripts/azure/tensor_configure.py | 84 +-
.../general/scripts/azure/zeppelin_configure.py | 82 +-
.../src/general/scripts/gcp/common_collect_data.py | 14 +-
.../general/scripts/gcp/common_create_bucket.py | 6 +-
.../general/scripts/gcp/common_create_firewall.py | 9 +-
.../general/scripts/gcp/common_create_instance.py | 5 +-
.../general/scripts/gcp/common_create_nat_route.py | 8 +-
.../scripts/gcp/common_create_notebook_image.py | 16 +-
.../scripts/gcp/common_create_service_account.py | 4 +-
.../general/scripts/gcp/common_create_subnet.py | 23 +-
.../scripts/gcp/common_download_git_certfile.py | 6 +-
...common_notebook_configure_dataengine-service.py | 21 +-
.../gcp/common_notebook_configure_dataengine.py | 29 +-
.../general/scripts/gcp/common_prepare_notebook.py | 26 +-
.../general/scripts/gcp/common_put_to_bucket.py | 2 +-
.../src/general/scripts/gcp/common_reupload_key.py | 7 +-
.../general/scripts/gcp/common_start_notebook.py | 23 +-
.../general/scripts/gcp/common_stop_notebook.py | 25 +-
.../scripts/gcp/common_terminate_notebook.py | 25 +-
.../scripts/gcp/common_terminate_notebook_image.py | 15 +-
.../scripts/gcp/dataengine-service_configure.py | 52 +-
.../scripts/gcp/dataengine-service_create.py | 10 +-
.../scripts/gcp/dataengine-service_install_libs.py | 12 +-
.../scripts/gcp/dataengine-service_list_libs.py | 8 +-
.../scripts/gcp/dataengine-service_prepare.py | 37 +-
.../scripts/gcp/dataengine-service_terminate.py | 24 +-
.../general/scripts/gcp/dataengine_configure.py | 91 +-
.../src/general/scripts/gcp/dataengine_prepare.py | 30 +-
.../src/general/scripts/gcp/dataengine_start.py | 25 +-
.../src/general/scripts/gcp/dataengine_stop.py | 21 +-
.../general/scripts/gcp/dataengine_terminate.py | 25 +-
.../general/scripts/gcp/deeplearning_configure.py | 69 +-
.../src/general/scripts/gcp/edge_configure.py | 63 +-
.../general/scripts/gcp/edge_create_static_ip.py | 10 +-
.../src/general/scripts/gcp/edge_start.py | 16 +-
.../src/general/scripts/gcp/edge_status.py | 20 +-
.../src/general/scripts/gcp/edge_stop.py | 17 +-
.../src/general/scripts/gcp/edge_terminate.py | 33 +-
.../src/general/scripts/gcp/jupyter_configure.py | 63 +-
.../jupyter_dataengine-service_create_configs.py | 17 +-
.../jupyter_install_dataengine-service_kernels.py | 31 +-
.../general/scripts/gcp/jupyterlab_configure.py | 73 +-
.../src/general/scripts/gcp/project_prepare.py | 38 +-
.../src/general/scripts/gcp/project_terminate.py | 35 +-
.../src/general/scripts/gcp/rstudio_configure.py | 73 +-
.../rstudio_dataengine-service_create_configs.py | 16 +-
.../rstudio_install_dataengine-service_kernels.py | 17 +-
.../src/general/scripts/gcp/ssn_configure.py | 70 +-
.../general/scripts/gcp/ssn_create_static_ip.py | 10 +-
.../src/general/scripts/gcp/ssn_create_vpc.py | 6 +-
.../src/general/scripts/gcp/ssn_finalize.py | 5 +-
.../src/general/scripts/gcp/ssn_prepare.py | 38 +-
.../src/general/scripts/gcp/ssn_terminate.py | 17 +-
.../scripts/gcp/ssn_terminate_gcp_resources.py | 7 +-
.../src/general/scripts/gcp/superset_configure.py | 76 +-
.../scripts/gcp/tensor-rstudio_configure.py | 76 +-
.../src/general/scripts/gcp/tensor_configure.py | 61 +-
.../src/general/scripts/gcp/zeppelin_configure.py | 61 +-
.../zeppelin_dataengine-service_create_configs.py | 18 +-
.../zeppelin_install_dataengine-service_kernels.py | 17 +-
.../src/general/scripts/os/check_inactivity.py | 11 +-
.../general/scripts/os/common_clean_instance.py | 4 +-
.../general/scripts/os/common_configure_proxy.py | 6 +-
.../scripts/os/common_configure_reverse_proxy.py | 2 +-
.../general/scripts/os/common_configure_spark.py | 14 +-
.../scripts/os/configure_proxy_for_docker.py | 7 +-
.../general/scripts/os/dataengine_install_libs.py | 12 +-
.../src/general/scripts/os/dataengine_list_libs.py | 8 +-
.../scripts/os/dataengine_reconfigure_spark.py | 12 +-
.../os/deeplearning_dataengine_create_configs.py | 15 +-
.../os/deeplearning_install_dataengine_kernels.py | 27 +-
.../general/scripts/os/get_list_available_pkgs.py | 9 +-
.../general/scripts/os/install_additional_libs.py | 9 +-
.../os/jupyter_dataengine_create_configs.py | 15 +-
.../os/jupyter_install_dataengine_kernels.py | 32 +-
.../scripts/os/jupyterlab_container_start.py | 5 +-
.../src/general/scripts/os/manage_git_creds.py | 12 +-
.../src/general/scripts/os/notebook_git_creds.py | 8 +-
.../scripts/os/notebook_inactivity_check.py | 14 +-
.../general/scripts/os/notebook_install_libs.py | 9 +-
.../src/general/scripts/os/notebook_list_libs.py | 8 +-
.../os/notebook_reconfigure_dataengine_spark.py | 15 +-
.../scripts/os/notebook_reconfigure_spark.py | 10 +-
.../src/general/scripts/os/reconfigure_spark.py | 11 +-
.../os/rstudio_dataengine_create_configs.py | 15 +-
.../os/rstudio_install_dataengine_kernels.py | 25 +-
.../src/general/scripts/os/superset_start.py | 7 +-
.../os/tensor-rstudio_dataengine_create_configs.py | 15 +-
.../tensor-rstudio_install_dataengine_kernels.py | 25 +-
.../scripts/os/tensor_dataengine_create_configs.py | 15 +-
.../os/tensor_install_dataengine_kernels.py | 27 +-
.../scripts/os/update_inactivity_on_start.py | 7 +-
.../os/zeppelin_dataengine_create_configs.py | 15 +-
.../os/zeppelin_install_dataengine_kernels.py | 29 +-
.../aws/jenkins_jobs/create_data_engine/config.xml | 18 +-
.../create_dataengine-service/config.xml | 48 +-
.../aws/jenkins_jobs/create_edge_node/config.xml | 7 +-
.../jenkins_jobs/create_notebook_image/config.xml | 7 +-
.../jenkins_jobs/create_notebook_server/config.xml | 8 +-
.../config.xml | 10 +-
.../config.xml | 7 +-
.../dataengine_install_additional_libs/config.xml | 9 +-
.../dataengine_list_available_libs/config.xml | 6 +-
.../dataengine_reconfigure_spark/config.xml | 9 +-
.../aws/jenkins_jobs/gitlab_server/config.xml | 29 +-
.../jenkins_jobs/manage_git_credentials/config.xml | 8 +-
.../notebook_install_additional_libs/config.xml | 5 +-
.../notebook_list_available_libs/config.xml | 6 +-
.../notebook_reconfigure_spark/config.xml | 7 +-
.../aws/jenkins_jobs/recreate_edge_node/config.xml | 7 +-
.../aws/jenkins_jobs/reupload_ssh_key/config.xml | 5 +-
.../aws/jenkins_jobs/start_data_engine/config.xml | 6 +-
.../aws/jenkins_jobs/start_edge_node/config.xml | 5 +-
.../jenkins_jobs/start_notebook_server/config.xml | 8 +-
.../aws/jenkins_jobs/stop_data_engine/config.xml | 6 +-
.../aws/jenkins_jobs/stop_edge_node/config.xml | 5 +-
.../jenkins_jobs/stop_notebook_server/config.xml | 6 +-
.../jenkins_jobs/terminate_data_engine/config.xml | 6 +-
.../terminate_dataengine-service/config.xml | 7 +-
.../jenkins_jobs/terminate_edge_node/config.xml | 5 +-
.../terminate_notebook_image/config.xml | 7 +-
.../terminate_notebook_server/config.xml | 6 +-
.../jenkins_jobs/create_data_engine/config.xml | 20 +-
.../azure/jenkins_jobs/create_edge_node/config.xml | 6 +-
.../jenkins_jobs/create_notebook_image/config.xml | 7 +-
.../jenkins_jobs/create_notebook_server/config.xml | 9 +-
.../dataengine_install_additional_libs/config.xml | 9 +-
.../dataengine_list_available_libs/config.xml | 6 +-
.../dataengine_reconfigure_spark/config.xml | 9 +-
.../jenkins_jobs/manage_git_credentials/config.xml | 8 +-
.../notebook_install_additional_libs/config.xml | 5 +-
.../notebook_list_available_libs/config.xml | 6 +-
.../notebook_reconfigure_spark/config.xml | 7 +-
.../jenkins_jobs/recreate_edge_node/config.xml | 7 +-
.../jenkins_jobs/start_data_engine/config.xml | 6 +-
.../azure/jenkins_jobs/start_edge_node/config.xml | 5 +-
.../jenkins_jobs/start_notebook_server/config.xml | 8 +-
.../azure/jenkins_jobs/stop_data_engine/config.xml | 6 +-
.../azure/jenkins_jobs/stop_edge_node/config.xml | 5 +-
.../jenkins_jobs/stop_notebook_server/config.xml | 6 +-
.../jenkins_jobs/terminate_data_engine/config.xml | 6 +-
.../jenkins_jobs/terminate_edge_node/config.xml | 5 +-
.../terminate_notebook_image/config.xml | 7 +-
.../terminate_notebook_server/config.xml | 6 +-
.../gcp/jenkins_jobs/create_data_engine/config.xml | 17 +-
.../create_dataengine-service/config.xml | 23 +-
.../gcp/jenkins_jobs/create_edge_node/config.xml | 6 +-
.../jenkins_jobs/create_notebook_server/config.xml | 8 +-
.../config.xml | 7 +-
.../config.xml | 7 +-
.../dataengine_install_additional_libs/config.xml | 9 +-
.../dataengine_list_available_libs/config.xml | 6 +-
.../dataengine_reconfigure_spark/config.xml | 9 +-
.../gcp/jenkins_jobs/gitlab_server/config.xml | 29 +-
.../jenkins_jobs/manage_git_credentials/config.xml | 8 +-
.../notebook_install_additional_libs/config.xml | 5 +-
.../notebook_list_available_libs/config.xml | 6 +-
.../notebook_reconfigure_spark/config.xml | 7 +-
.../gcp/jenkins_jobs/recreate_edge_node/config.xml | 6 +-
.../gcp/jenkins_jobs/start_data_engine/config.xml | 6 +-
.../gcp/jenkins_jobs/start_edge_node/config.xml | 5 +-
.../jenkins_jobs/start_notebook_server/config.xml | 7 +-
.../gcp/jenkins_jobs/stop_data_engine/config.xml | 6 +-
.../gcp/jenkins_jobs/stop_edge_node/config.xml | 5 +-
.../jenkins_jobs/stop_notebook_server/config.xml | 6 +-
.../jenkins_jobs/terminate_data_engine/config.xml | 6 +-
.../terminate_dataengine-service/config.xml | 8 +-
.../jenkins_jobs/terminate_edge_node/config.xml | 5 +-
.../terminate_notebook_server/config.xml | 6 +-
.../templates/os/debian/spark-master.service | 2 +-
.../templates/os/debian/spark-slave.service | 2 +-
.../templates/os/redhat/spark-master.service | 2 +-
.../templates/os/redhat/spark-slave.service | 2 +-
.../src/general/templates/os/renew_certificates.sh | 6 +-
infrastructure-provisioning/src/jupyter/fabfile.py | 12 +-
.../src/jupyter/scripts/configure_jupyter_node.py | 9 +-
.../src/jupyterlab/fabfile.py | 12 +-
.../scripts/configure_jupyterlab_node.py | 9 +-
infrastructure-provisioning/src/project/fabfile.py | 9 +-
.../src/project/scripts/configure_http_proxy.py | 5 +-
.../src/project/scripts/configure_keycloak.py | 11 +-
.../src/project/scripts/configure_nftables.py | 5 +-
.../scripts/configure_nginx_reverse_proxy.py | 10 +-
.../src/project/scripts/reupload_ssh_key.py | 9 +-
.../src/project/templates/conf.d/proxy.conf | 4 +-
.../src/project/templates/squid.conf | 4 +-
infrastructure-provisioning/src/rstudio/fabfile.py | 12 +-
.../src/rstudio/scripts/configure_rstudio_node.py | 15 +-
infrastructure-provisioning/src/ssn/fabfile.py | 5 +-
.../src/ssn/files/aws/mongo_roles.json | 18 +-
.../src/ssn/files/azure/mongo_roles.json | 12 +-
.../src/ssn/files/gcp/mongo_roles.json | 20 +-
.../src/ssn/scripts/backup.py | 61 +-
.../src/ssn/scripts/configure_billing.py | 53 +-
.../src/ssn/scripts/configure_conf_file.py | 19 +-
.../src/ssn/scripts/configure_docker.py | 83 +-
.../src/ssn/scripts/configure_gitlab.py | 26 +-
.../src/ssn/scripts/configure_mongo.py | 25 +-
.../src/ssn/scripts/configure_ssn_node.py | 100 +-
.../src/ssn/scripts/configure_ui.py | 95 +-
.../src/ssn/scripts/docker_build.py | 29 +-
.../src/ssn/scripts/resource_status.py | 8 +-
.../src/ssn/scripts/restore.py | 47 +-
.../src/ssn/scripts/upload_response_file.py | 10 +-
.../src/ssn/templates/gitlab.ini | 20 +-
.../src/ssn/templates/nginx_proxy.conf | 4 +-
.../src/ssn/templates/ssn.yml | 4 +-
.../src/ssn/templates/supervisor_svc.conf | 8 +-
.../src/superset/fabfile.py | 12 +-
.../superset/scripts/configure_superset_node.py | 11 +-
.../src/tensor-rstudio/fabfile.py | 12 +-
.../scripts/configure_tensor-rstudio_node.py | 15 +-
infrastructure-provisioning/src/tensor/fabfile.py | 12 +-
.../src/tensor/scripts/configure_tensor_node.py | 15 +-
.../src/zeppelin/fabfile.py | 12 +-
.../zeppelin/scripts/configure_zeppelin_node.py | 14 +-
.../terraform/aws/endpoint/main/README.md | 2 +-
.../terraform/aws/endpoint/main/variables.tf | 2 +-
.../terraform/aws/project/main/variales.tf | 2 +-
.../main/datalab-billing-chart}/.helmignore | 0
.../main/datalab-billing-chart/Chart.yaml | 26 +
.../main/datalab-billing-chart/templates/NOTES.txt | 42 +
.../datalab-billing-chart/templates/_helpers.tpl | 65 +
.../templates/configmap-billing-conf.yaml | 105 ++
.../templates/deployment.yaml | 86 ++
.../datalab-billing-chart/templates/service.yaml | 38 +
.../main/datalab-billing-chart/values.yaml | 76 ++
.../aws/ssn-helm-charts/main/datalab-billing.tf | 59 +
.../main/datalab-ui-chart}/.helmignore | 0
.../main/datalab-ui-chart/Chart.yaml | 26 +
.../main/datalab-ui-chart/templates/NOTES.txt | 42 +
.../main/datalab-ui-chart/templates/_helpers.tpl | 65 +
.../main/datalab-ui-chart/templates/cert.yaml | 64 +
.../templates/configmap-ui-conf.yaml | 235 ++++
.../datalab-ui-chart/templates/deployment.yaml | 107 ++
.../main/datalab-ui-chart/templates/ingress.yaml | 57 +
.../main/datalab-ui-chart/templates/service.yaml | 43 +
.../main/datalab-ui-chart/values.yaml | 69 +
.../aws/ssn-helm-charts/main/datalab-ui.tf | 73 ++
.../main/dlab-billing-chart/Chart.yaml | 26 -
.../main/dlab-billing-chart/templates/NOTES.txt | 42 -
.../main/dlab-billing-chart/templates/_helpers.tpl | 65 -
.../templates/configmap-billing-conf.yaml | 105 --
.../dlab-billing-chart/templates/deployment.yaml | 86 --
.../main/dlab-billing-chart/templates/service.yaml | 38 -
.../main/dlab-billing-chart/values.yaml | 76 --
.../aws/ssn-helm-charts/main/dlab-billing.tf | 57 -
.../ssn-helm-charts/main/dlab-ui-chart/Chart.yaml | 26 -
.../main/dlab-ui-chart/templates/NOTES.txt | 42 -
.../main/dlab-ui-chart/templates/_helpers.tpl | 65 -
.../main/dlab-ui-chart/templates/cert.yaml | 64 -
.../dlab-ui-chart/templates/configmap-ui-conf.yaml | 235 ----
.../main/dlab-ui-chart/templates/deployment.yaml | 107 --
.../main/dlab-ui-chart/templates/ingress.yaml | 57 -
.../main/dlab-ui-chart/templates/service.yaml | 43 -
.../ssn-helm-charts/main/dlab-ui-chart/values.yaml | 69 -
.../terraform/aws/ssn-helm-charts/main/dlab-ui.tf | 70 -
.../main/files/configure_keycloak.sh | 10 +-
.../main/files/keycloak_values.yaml | 8 +-
.../terraform/aws/ssn-helm-charts/main/keycloak.tf | 13 +-
.../terraform/aws/ssn-helm-charts/main/main.tf | 6 +-
.../terraform/aws/ssn-helm-charts/main/mongo.tf | 6 +-
.../terraform/aws/ssn-helm-charts/main/mysql.tf | 12 +-
.../terraform/aws/ssn-helm-charts/main/nginx.tf | 6 +-
.../terraform/aws/ssn-helm-charts/main/secrets.tf | 36 +-
.../ssn-helm-charts/main/step-ca-chart/values.yaml | 2 +-
.../terraform/aws/ssn-helm-charts/main/step-ca.tf | 11 +-
.../aws/ssn-helm-charts/main/step-issuer.tf | 4 +-
.../aws/ssn-helm-charts/main/variables.tf | 16 +-
.../terraform/aws/ssn-k8s/main/README.md | 6 +-
.../aws/ssn-k8s/main/files/masters-user-data.sh | 8 +-
.../aws/ssn-k8s/main/files/workers-user-data.sh | 2 +-
.../terraform/aws/ssn-k8s/main/variables.tf | 6 +-
.../terraform/azure/endpoint/main/variables.tf | 2 +-
.../terraform/bin/datalab.py | 1358 ++++++++++++++++++++
.../terraform/bin/deploy/billing_app_aws.yml | 6 +-
.../terraform/bin/deploy/billing_aws.yml | 16 +-
.../terraform/bin/deploy/billing_azure.yml | 10 +-
.../terraform/bin/deploy/billing_gcp.yml | 8 +-
.../terraform/bin/deploy/endpoint_fab.py | 135 +-
.../bin/deploy/mongo_files/configure_mongo.py | 13 +-
.../bin/deploy/mongo_files/gcp/mongo_roles.json | 20 +-
.../terraform/bin/deploy/provisioning.yml | 18 +-
.../terraform/bin/deploy/renew_certificates.sh | 6 +-
.../terraform/bin/deploy/supervisor_svc.conf | 4 +-
infrastructure-provisioning/terraform/bin/dlab.py | 1358 --------------------
.../terraform/bin/terraform-cli.py | 10 +-
.../terraform/gcp/endpoint/main/variables.tf | 4 +-
.../terraform/gcp/endpoint/provisioning.py | 234 ++--
.../terraform/gcp/endpoint/provisioning.yml | 24 +-
.../terraform/gcp/endpoint/supervisor_svc.conf | 2 +-
.../helm_charts/datalab-billing-chart}/.helmignore | 0
.../helm_charts/datalab-billing-chart/Chart.yaml | 26 +
.../datalab-billing-chart/templates/NOTES.txt | 42 +
.../datalab-billing-chart/templates/_helpers.tpl | 65 +
.../templates/configmap-billing-conf.yaml | 49 +
.../templates/deployment.yaml | 86 ++
.../datalab-billing-chart/templates/service.yaml | 38 +
.../helm_charts/datalab-billing-chart/values.yaml | 63 +
.../main/modules/helm_charts/datalab-billing.tf | 47 +
.../helm_charts/datalab-ui-chart}/.helmignore | 0
.../helm_charts/datalab-ui-chart/Chart.yaml | 26 +
.../datalab-ui-chart/templates/NOTES.txt | 42 +
.../datalab-ui-chart/templates/_helpers.tpl | 65 +
.../datalab-ui-chart/templates/cert.yaml | 64 +
.../templates/configmap-ui-conf.yaml | 235 ++++
.../datalab-ui-chart/templates/deployment.yaml | 107 ++
.../datalab-ui-chart/templates/ingress.yaml | 57 +
.../datalab-ui-chart/templates/service.yaml | 43 +
.../helm_charts/datalab-ui-chart/values.yaml | 66 +
.../ssn-gke/main/modules/helm_charts/datalab-ui.tf | 74 ++
.../helm_charts/dlab-billing-chart/Chart.yaml | 26 -
.../dlab-billing-chart/templates/NOTES.txt | 42 -
.../dlab-billing-chart/templates/_helpers.tpl | 65 -
.../templates/configmap-billing-conf.yaml | 49 -
.../dlab-billing-chart/templates/deployment.yaml | 86 --
.../dlab-billing-chart/templates/service.yaml | 38 -
.../helm_charts/dlab-billing-chart/values.yaml | 63 -
.../main/modules/helm_charts/dlab-billing.tf | 44 -
.../modules/helm_charts/dlab-ui-chart/Chart.yaml | 26 -
.../helm_charts/dlab-ui-chart/templates/NOTES.txt | 42 -
.../dlab-ui-chart/templates/_helpers.tpl | 65 -
.../helm_charts/dlab-ui-chart/templates/cert.yaml | 64 -
.../dlab-ui-chart/templates/configmap-ui-conf.yaml | 235 ----
.../dlab-ui-chart/templates/deployment.yaml | 107 --
.../dlab-ui-chart/templates/ingress.yaml | 57 -
.../dlab-ui-chart/templates/service.yaml | 43 -
.../modules/helm_charts/dlab-ui-chart/values.yaml | 66 -
.../ssn-gke/main/modules/helm_charts/dlab-ui.tf | 70 -
.../main/modules/helm_charts/external-dns.tf | 10 +-
.../external-dns/templates/externaldns.yaml | 2 +-
.../helm_charts/files/configure_keycloak.sh | 10 +-
.../helm_charts/files/get_configmap_values.sh | 6 +-
.../modules/helm_charts/files/keycloak_values.yaml | 8 +-
.../ssn-gke/main/modules/helm_charts/keycloak.tf | 13 +-
.../gcp/ssn-gke/main/modules/helm_charts/main.tf | 2 +-
.../gcp/ssn-gke/main/modules/helm_charts/mongo.tf | 10 +-
.../gcp/ssn-gke/main/modules/helm_charts/mysql.tf | 6 +-
.../nginx-ingress/templates/nginx-ingress.yaml | 4 +-
.../gcp/ssn-gke/main/modules/helm_charts/nginx.tf | 16 +-
.../ssn-gke/main/modules/helm_charts/outputs.tf | 2 +-
.../ssn-gke/main/modules/helm_charts/secrets.tf | 36 +-
.../modules/helm_charts/step-ca-chart/values.yaml | 2 +-
.../ssn-gke/main/modules/helm_charts/step-ca.tf | 10 +-
.../main/modules/helm_charts/step-issuer.tf | 4 +-
.../terraform/gcp/ssn-gke/main/variables.tf | 14 +-
.../terraform/keycloak-theme/Dockerfile | 2 +-
integration-tests-cucumber/pom.xml | 2 +-
.../java/org/apache/datalab/dto/EndpointDTO.java | 38 +
.../org/apache/datalab/mongo/MongoDBHelper.java | 33 +
.../org/apache/datalab/util/JacksonMapper.java | 35 +
.../org/apache/datalab/util/PropertyHelper.java | 42 +
.../main/java/org/apache/dlab/dto/EndpointDTO.java | 38 -
.../java/org/apache/dlab/mongo/MongoDBHelper.java | 33 -
.../java/org/apache/dlab/util/JacksonMapper.java | 35 -
.../java/org/apache/dlab/util/PropertyHelper.java | 42 -
.../src/test/java/datalab/Constants.java | 24 +
.../src/test/java/datalab/RunCucumberTest.java | 29 +
.../test/java/datalab/endpoint/EndpointSteps.java | 116 ++
.../src/test/java/datalab/login/LoginSteps.java | 63 +
.../src/test/java/dlab/Constants.java | 24 -
.../src/test/java/dlab/RunCucumberTest.java | 29 -
.../src/test/java/dlab/endpoint/EndpointSteps.java | 116 --
.../src/test/java/dlab/login/LoginSteps.java | 63 -
.../src/test/resources/config.properties | 4 +-
.../src/test/resources/datalab/endpoint.feature | 75 ++
.../src/test/resources/datalab/login.feature | 30 +
.../src/test/resources/dlab/endpoint.feature | 75 --
.../src/test/resources/dlab/login.feature | 30 -
integration-tests/README.MD | 4 +-
integration-tests/pom.xml | 28 +-
.../cloud/VirtualMachineStatusChecker.java | 18 +-
.../dlab/automation/cloud/aws/AmazonHelper.java | 8 +-
.../automation/cloud/aws/AmazonInstanceState.java | 2 +-
.../dlab/automation/cloud/azure/AzureHelper.java | 6 +-
.../epam/dlab/automation/cloud/gcp/GcpHelper.java | 6 +-
.../automation/cloud/gcp/GcpInstanceState.java | 2 +-
.../com/epam/dlab/automation/docker/AckStatus.java | 2 +-
.../com/epam/dlab/automation/docker/Bridge.java | 2 +-
.../com/epam/dlab/automation/docker/Docker.java | 6 +-
.../dlab/automation/docker/DockerContainer.java | 2 +-
.../epam/dlab/automation/docker/HostConfig.java | 2 +-
.../com/epam/dlab/automation/docker/Labels.java | 2 +-
.../dlab/automation/docker/NetworkSettings.java | 2 +-
.../com/epam/dlab/automation/docker/Networks.java | 2 +-
.../epam/dlab/automation/docker/SSHConnect.java | 4 +-
.../dlab/automation/exceptions/CloudException.java | 2 +-
.../automation/exceptions/DockerException.java | 2 +-
.../automation/exceptions/JenkinsException.java | 2 +-
.../automation/exceptions/LoadFailException.java | 2 +-
.../epam/dlab/automation/helper/CloudHelper.java | 18 +-
.../epam/dlab/automation/helper/CloudProvider.java | 2 +-
.../automation/helper/ConfigPropertyValue.java | 4 +-
.../epam/dlab/automation/helper/NamingHelper.java | 2 +-
.../dlab/automation/helper/PropertiesResolver.java | 2 +-
.../epam/dlab/automation/helper/WaitForStatus.java | 8 +-
.../com/epam/dlab/automation/http/ApiPath.java | 2 +-
.../com/epam/dlab/automation/http/ContentType.java | 2 +-
.../com/epam/dlab/automation/http/HttpRequest.java | 4 +-
.../epam/dlab/automation/http/HttpStatusCode.java | 2 +-
.../jenkins/JenkinsConfigProperties.java | 2 +-
.../jenkins/JenkinsResponseElements.java | 2 +-
.../dlab/automation/jenkins/JenkinsService.java | 10 +-
.../epam/dlab/automation/jenkins/JenkinsUrls.java | 2 +-
.../dlab/automation/model/CreateNotebookDto.java | 2 +-
.../dlab/automation/model/DeployClusterDto.java | 2 +-
.../dlab/automation/model/DeployDataProcDto.java | 2 +-
.../epam/dlab/automation/model/DeployEMRDto.java | 2 +-
.../epam/dlab/automation/model/DeploySparkDto.java | 2 +-
.../dlab/automation/model/ExploratoryImageDto.java | 2 +-
.../com/epam/dlab/automation/model/ImageDto.java | 2 +-
.../epam/dlab/automation/model/JsonMapperDto.java | 2 +-
.../java/com/epam/dlab/automation/model/Lib.java | 2 +-
.../com/epam/dlab/automation/model/LoginDto.java | 2 +-
.../epam/dlab/automation/model/NotebookConfig.java | 2 +-
integration-tests/src/main/resources/log4j2.xml | 18 +-
.../epam/dlab/automation/test/TestCallable.java | 32 +-
.../automation/test/TestDataEngineService.java | 16 +-
.../epam/dlab/automation/test/TestServices.java | 26 +-
.../test/libs/LibraryNotFoundException.java | 2 +-
.../epam/dlab/automation/test/libs/LibsHelper.java | 6 +-
.../dlab/automation/test/libs/TestDescription.java | 2 +-
.../automation/test/libs/TestLibGroupStep.java | 12 +-
.../automation/test/libs/TestLibInstallStep.java | 22 +-
.../dlab/automation/test/libs/TestLibListStep.java | 20 +-
.../dlab/automation/test/libs/TestLibStep.java | 2 +-
.../test/libs/models/LibInstallRequest.java | 4 +-
.../test/libs/models/LibSearchRequest.java | 2 +-
.../test/libs/models/LibStatusResponse.java | 2 +-
.../test/libs/models/LibToSearchData.java | 2 +-
.../automation/test/libs/models/LibraryStatus.java | 2 +-
integration-tests/src/test/resources/log4j2.xml | 18 +-
pom.xml | 56 +-
services/billing-aws/billing.yml | 16 +-
services/billing-aws/pom.xml | 10 +-
.../com/epam/datalab/BillingAwsApplication.java | 37 +
.../main/java/com/epam/datalab/BillingService.java | 29 +
.../java/com/epam/datalab/BillingServiceImpl.java | 127 ++
.../main/java/com/epam/datalab/BillingTool.java | 176 +++
.../src/main/java/com/epam/datalab/Help.java | 146 +++
.../java/com/epam/datalab/conf/SecurityConfig.java | 66 +
.../configuration/BillingToolConfiguration.java | 282 ++++
.../BillingToolConfigurationFactory.java | 110 ++
.../datalab/configuration/ConfigJsonGenerator.java | 103 ++
.../configuration/ConfigurationValidator.java | 71 +
.../datalab/configuration/FreeMarkerConfig.java | 84 ++
.../configuration/LoggingConfigurationFactory.java | 170 +++
.../epam/datalab/controller/BillingController.java | 44 +
.../java/com/epam/datalab/core/AdapterBase.java | 173 +++
.../java/com/epam/datalab/core/BillingUtils.java | 144 +++
.../java/com/epam/datalab/core/DBAdapterBase.java | 163 +++
.../java/com/epam/datalab/core/FilterBase.java | 95 ++
.../java/com/epam/datalab/core/ModuleBase.java | 84 ++
.../java/com/epam/datalab/core/ModuleData.java | 182 +++
.../java/com/epam/datalab/core/ModuleType.java | 46 +
.../core/aggregate/AggregateGranularity.java | 44 +
.../datalab/core/aggregate/DataAggregator.java | 172 +++
.../epam/datalab/core/aggregate/UsageDataList.java | 87 ++
.../com/epam/datalab/core/parser/ColumnInfo.java | 59 +
.../com/epam/datalab/core/parser/ColumnMeta.java | 318 +++++
.../com/epam/datalab/core/parser/CommonFormat.java | 304 +++++
.../datalab/core/parser/ConditionEvaluate.java | 168 +++
.../com/epam/datalab/core/parser/ParserBase.java | 395 ++++++
.../com/epam/datalab/core/parser/ParserByLine.java | 249 ++++
.../epam/datalab/core/parser/ParserStatistics.java | 194 +++
.../com/epam/datalab/logging/AppenderBase.java | 103 ++
.../com/epam/datalab/logging/AppenderConsole.java | 44 +
.../com/epam/datalab/logging/AppenderFile.java | 208 +++
.../com/epam/datalab/module/AdapterConsole.java | 92 ++
.../java/com/epam/datalab/module/AdapterFile.java | 164 +++
.../java/com/epam/datalab/module/ModuleName.java | 33 +
.../java/com/epam/datalab/module/ParserCsv.java | 312 +++++
.../com/epam/datalab/module/aws/AdapterS3File.java | 388 ++++++
.../com/epam/datalab/module/aws/FilterAWS.java | 150 +++
.../com/epam/datalab/module/aws/S3FileList.java | 168 +++
.../com/epam/datalab/mongo/AdapterMongoDb.java | 229 ++++
.../epam/datalab/mongo/DatalabResourceTypeDAO.java | 115 ++
.../com/epam/datalab/mongo/MongoConstants.java | 32 +
.../com/epam/datalab/mongo/MongoDbConnection.java | 223 ++++
.../java/com/epam/dlab/BillingAwsApplication.java | 37 -
.../main/java/com/epam/dlab/BillingService.java | 29 -
.../java/com/epam/dlab/BillingServiceImpl.java | 127 --
.../src/main/java/com/epam/dlab/BillingTool.java | 164 ---
.../src/main/java/com/epam/dlab/Help.java | 136 --
.../java/com/epam/dlab/conf/SecurityConfig.java | 66 -
.../configuration/BillingToolConfiguration.java | 282 ----
.../BillingToolConfigurationFactory.java | 101 --
.../dlab/configuration/ConfigJsonGenerator.java | 88 --
.../dlab/configuration/ConfigurationValidator.java | 63 -
.../epam/dlab/configuration/FreeMarkerConfig.java | 81 --
.../configuration/LoggingConfigurationFactory.java | 145 ---
.../epam/dlab/controller/BillingController.java | 44 -
.../main/java/com/epam/dlab/core/AdapterBase.java | 173 ---
.../main/java/com/epam/dlab/core/BillingUtils.java | 129 --
.../java/com/epam/dlab/core/DBAdapterBase.java | 127 --
.../main/java/com/epam/dlab/core/FilterBase.java | 95 --
.../main/java/com/epam/dlab/core/ModuleBase.java | 73 --
.../main/java/com/epam/dlab/core/ModuleData.java | 182 ---
.../main/java/com/epam/dlab/core/ModuleType.java | 45 -
.../dlab/core/aggregate/AggregateGranularity.java | 42 -
.../epam/dlab/core/aggregate/DataAggregator.java | 172 ---
.../epam/dlab/core/aggregate/UsageDataList.java | 75 --
.../java/com/epam/dlab/core/parser/ColumnInfo.java | 50 -
.../java/com/epam/dlab/core/parser/ColumnMeta.java | 318 -----
.../com/epam/dlab/core/parser/CommonFormat.java | 304 -----
.../epam/dlab/core/parser/ConditionEvaluate.java | 152 ---
.../java/com/epam/dlab/core/parser/ParserBase.java | 311 -----
.../com/epam/dlab/core/parser/ParserByLine.java | 249 ----
.../epam/dlab/core/parser/ParserStatistics.java | 153 ---
.../java/com/epam/dlab/logging/AppenderBase.java | 93 --
.../com/epam/dlab/logging/AppenderConsole.java | 44 -
.../java/com/epam/dlab/logging/AppenderFile.java | 181 ---
.../java/com/epam/dlab/module/AdapterConsole.java | 92 --
.../java/com/epam/dlab/module/AdapterFile.java | 164 ---
.../main/java/com/epam/dlab/module/ModuleName.java | 32 -
.../main/java/com/epam/dlab/module/ParserCsv.java | 272 ----
.../com/epam/dlab/module/aws/AdapterS3File.java | 388 ------
.../java/com/epam/dlab/module/aws/FilterAWS.java | 150 ---
.../java/com/epam/dlab/module/aws/S3FileList.java | 168 ---
.../java/com/epam/dlab/mongo/AdapterMongoDb.java | 248 ----
.../com/epam/dlab/mongo/DlabResourceTypeDAO.java | 189 ---
.../java/com/epam/dlab/mongo/MongoConstants.java | 59 -
.../com/epam/dlab/mongo/MongoDbConnection.java | 223 ----
.../billing-aws/src/main/resources/application.yml | 6 +-
...b.configuration.BillingToolConfigurationFactory | 8 +
.../src/main/resources/com.epam.dlab.Help.conf.txt | 2 +-
...b.configuration.BillingToolConfigurationFactory | 8 -
.../BillingToolConfigurationTest.java | 97 ++
.../configuration/ConfigJsonGeneratorTest.java | 59 +
.../configuration/ConfigurationValidatorTest.java | 54 +
.../LoggingConfigurationFactoryTest.java | 89 ++
.../com/epam/datalab/core/BillingUtilsTest.java | 107 ++
.../datalab/core/aggregate/DataAggregatorTest.java | 47 +
.../datalab/core/aggregate/UsageDataListTest.java | 116 ++
.../core/parser/BillingResourceTypeTest.java | 36 +
.../epam/datalab/core/parser/ColumnInfoTest.java | 37 +
.../epam/datalab/core/parser/ColumnMetaTest.java | 86 ++
.../epam/datalab/core/parser/CommonFormatTest.java | 107 ++
.../datalab/core/parser/ConditionEvaluateTest.java | 118 ++
.../epam/datalab/core/parser/ReportLineTest.java | 94 ++
.../epam/datalab/logging/AppenderConsoleTest.java | 33 +
.../com/epam/datalab/logging/AppenderFileTest.java | 42 +
.../epam/datalab/module/AdapterConsoleTest.java | 33 +
.../com/epam/datalab/module/AdapterFileTest.java | 41 +
.../epam/datalab/module/AdapterMongoDBTest.java | 55 +
.../com/epam/datalab/module/ParserCsvTest.java | 81 ++
.../epam/datalab/module/aws/AdapterS3FileTest.java | 50 +
.../com/epam/datalab/module/aws/FilterAWSTest.java | 50 +
.../epam/datalab/module/aws/S3FileListTest.java | 125 ++
.../BillingToolConfigurationTest.java | 98 --
.../configuration/ConfigJsonGeneratorTest.java | 60 -
.../configuration/ConfigurationValidatorTest.java | 55 -
.../LoggingConfigurationFactoryTest.java | 91 --
.../java/com/epam/dlab/core/BillingUtilsTest.java | 108 --
.../dlab/core/aggregate/DataAggregatorTest.java | 47 -
.../dlab/core/aggregate/UsageDataListTest.java | 116 --
.../dlab/core/parser/BillingResourceTypeTest.java | 36 -
.../com/epam/dlab/core/parser/ColumnInfoTest.java | 37 -
.../com/epam/dlab/core/parser/ColumnMetaTest.java | 86 --
.../epam/dlab/core/parser/CommonFormatTest.java | 107 --
.../dlab/core/parser/ConditionEvaluateTest.java | 119 --
.../com/epam/dlab/core/parser/ReportLineTest.java | 94 --
.../com/epam/dlab/logging/AppenderConsoleTest.java | 33 -
.../com/epam/dlab/logging/AppenderFileTest.java | 42 -
.../com/epam/dlab/module/AdapterConsoleTest.java | 33 -
.../java/com/epam/dlab/module/AdapterFileTest.java | 42 -
.../com/epam/dlab/module/AdapterMongoDBTest.java | 56 -
.../java/com/epam/dlab/module/ParserCsvTest.java | 82 --
.../epam/dlab/module/aws/AdapterS3FileTest.java | 51 -
.../com/epam/dlab/module/aws/FilterAWSTest.java | 50 -
.../com/epam/dlab/module/aws/S3FileListTest.java | 126 --
services/billing-azure/billing.yml | 10 +-
services/billing-azure/pom.xml | 8 +-
.../azure/AzureInvoiceCalculationService.java | 242 ++++
.../billing/azure/BillingAzureApplication.java | 36 +
.../billing/azure/CalculateBillingService.java | 29 +
.../billing/azure/CalculateBillingServiceImpl.java | 247 ++++
.../billing/azure/MongoDbBillingClient.java | 50 +
.../epam/datalab/billing/azure/MongoDocument.java | 70 +
.../config/AggregationOutputMongoDataSource.java | 31 +
.../azure/config/BillingConfigurationAzure.java | 52 +
.../azure/config/LoggingConfigurationFactory.java | 166 +++
.../billing/azure/config/SecurityConfig.java | 66 +
.../azure/controller/BillingController.java | 44 +
.../billing/azure/logging/AppenderBase.java | 103 ++
.../billing/azure/logging/AppenderConsole.java | 44 +
.../billing/azure/logging/AppenderFile.java | 200 +++
.../azure/model/AzureDailyResourceInvoice.java | 48 +
.../azure/model/AzureDatalabBillableResource.java | 39 +
.../datalab/billing/azure/model/BillingPeriod.java | 35 +
.../billing/azure/rate/AzureRateCardClient.java | 109 ++
.../com/epam/datalab/billing/azure/rate/Meter.java | 52 +
.../billing/azure/rate/RateCardResponse.java | 41 +
.../azure/usage/AzureUsageAggregateClient.java | 103 ++
.../datalab/billing/azure/usage/InstanceData.java | 29 +
.../billing/azure/usage/MicrosoftResources.java | 37 +
.../billing/azure/usage/UsageAggregateRecord.java | 35 +
.../usage/UsageAggregateRecordProperties.java | 54 +
.../azure/usage/UsageAggregateResponse.java | 33 +
.../azure/AzureInvoiceCalculationService.java | 242 ----
.../billing/azure/BillingAzureApplication.java | 36 -
.../billing/azure/CalculateBillingService.java | 29 -
.../billing/azure/CalculateBillingServiceImpl.java | 247 ----
.../dlab/billing/azure/MongoDbBillingClient.java | 50 -
.../com/epam/dlab/billing/azure/MongoDocument.java | 70 -
.../config/AggregationOutputMongoDataSource.java | 31 -
.../azure/config/BillingConfigurationAzure.java | 52 -
.../azure/config/LoggingConfigurationFactory.java | 166 ---
.../dlab/billing/azure/config/SecurityConfig.java | 66 -
.../azure/controller/BillingController.java | 44 -
.../dlab/billing/azure/logging/AppenderBase.java | 92 --
.../billing/azure/logging/AppenderConsole.java | 43 -
.../dlab/billing/azure/logging/AppenderFile.java | 173 ---
.../azure/model/AzureDailyResourceInvoice.java | 48 -
.../azure/model/AzureDlabBillableResource.java | 39 -
.../dlab/billing/azure/model/BillingPeriod.java | 35 -
.../billing/azure/rate/AzureRateCardClient.java | 105 --
.../com/epam/dlab/billing/azure/rate/Meter.java | 52 -
.../dlab/billing/azure/rate/RateCardResponse.java | 41 -
.../azure/usage/AzureUsageAggregateClient.java | 103 --
.../dlab/billing/azure/usage/InstanceData.java | 29 -
.../billing/azure/usage/MicrosoftResources.java | 37 -
.../billing/azure/usage/UsageAggregateRecord.java | 35 -
.../usage/UsageAggregateRecordProperties.java | 54 -
.../azure/usage/UsageAggregateResponse.java | 33 -
.../src/main/resources/application.yml | 18 +-
services/billing-gcp/billing.yml | 8 +-
services/billing-gcp/pom.xml | 8 +-
.../datalab/billing/gcp/BillingGcpApplication.java | 36 +
.../gcp/conf/BillingApplicationConfiguration.java | 34 +
.../billing/gcp/conf/DatalabConfiguration.java | 35 +
.../datalab/billing/gcp/conf/SecurityConfig.java | 66 +
.../billing/gcp/controller/BillingController.java | 44 +
.../epam/datalab/billing/gcp/dao/BillingDAO.java | 29 +
.../billing/gcp/dao/impl/BigQueryBillingDAO.java | 131 ++
.../datalab/billing/gcp/model/BillingHistory.java | 32 +
.../gcp/repository/BillingHistoryRepository.java | 26 +
.../billing/gcp/service/BillingService.java | 29 +
.../gcp/service/impl/BillingServiceImpl.java | 52 +
.../dlab/billing/gcp/BillingGcpApplication.java | 36 -
.../gcp/conf/BillingApplicationConfiguration.java | 34 -
.../dlab/billing/gcp/conf/DlabConfiguration.java | 35 -
.../epam/dlab/billing/gcp/conf/SecurityConfig.java | 66 -
.../billing/gcp/controller/BillingController.java | 44 -
.../com/epam/dlab/billing/gcp/dao/BillingDAO.java | 29 -
.../billing/gcp/dao/impl/BigQueryBillingDAO.java | 131 --
.../dlab/billing/gcp/model/BillingHistory.java | 32 -
.../dlab/billing/gcp/model/GcpBillingData.java | 44 -
.../gcp/repository/BillingHistoryRepository.java | 26 -
.../billing/gcp/repository/BillingRepository.java | 27 -
.../dlab/billing/gcp/service/BillingService.java | 29 -
.../gcp/service/impl/BillingServiceImpl.java | 52 -
.../billing-gcp/src/main/resources/application.yml | 16 +-
services/common/pom.xml | 4 +-
.../datalab/billing/BillingCalculationUtils.java | 42 +
.../epam/datalab/billing/DatalabResourceType.java | 54 +
.../epam/datalab/exceptions/AdapterException.java | 56 +
.../exceptions/DatalabAuthenticationException.java | 26 +
.../epam/datalab/exceptions/DatalabException.java | 34 +
.../exceptions/DatalabValidationException.java | 26 +
.../epam/datalab/exceptions/GenericException.java | 56 +
.../exceptions/InitializationException.java | 56 +
.../epam/datalab/exceptions/ParseException.java | 57 +
.../exceptions/ResourceAlreadyExistException.java | 26 +
.../exceptions/ResourceConflictException.java | 26 +
.../ResourceInappropriateStateException.java | 26 +
.../exceptions/ResourceNotFoundException.java | 26 +
.../exceptions/ResourceQuoteReachedException.java | 29 +
.../epam/dlab/billing/BillingCalculationUtils.java | 42 -
.../com/epam/dlab/billing/DlabResourceType.java | 54 -
.../com/epam/dlab/exceptions/AdapterException.java | 49 -
.../exceptions/DlabAuthenticationException.java | 26 -
.../com/epam/dlab/exceptions/DlabException.java | 34 -
.../dlab/exceptions/DlabValidationException.java | 26 -
.../com/epam/dlab/exceptions/GenericException.java | 49 -
.../dlab/exceptions/InitializationException.java | 49 -
.../com/epam/dlab/exceptions/ParseException.java | 50 -
.../exceptions/ResourceAlreadyExistException.java | 26 -
.../dlab/exceptions/ResourceConflictException.java | 26 -
.../ResourceInappropriateStateException.java | 26 -
.../dlab/exceptions/ResourceNotFoundException.java | 26 -
.../exceptions/ResourceQuoteReachedException.java | 29 -
services/datalab-model/pom.xml | 79 ++
.../main/java/com/epam/datalab/MongoKeyWords.java | 34 +
.../datalab/dto/InfrastructureMetaInfoDTO.java | 34 +
.../epam/datalab/dto/LibListComputationalDTO.java | 55 +
.../epam/datalab/dto/LibListExploratoryDTO.java | 40 +
.../java/com/epam/datalab/dto/LibraryGroups.java | 40 +
.../java/com/epam/datalab/dto/ResourceBaseDTO.java | 74 ++
.../com/epam/datalab/dto/ResourceEnvBaseDTO.java | 71 +
.../com/epam/datalab/dto/ResourceSysBaseDTO.java | 87 ++
.../java/com/epam/datalab/dto/ResourceURL.java | 38 +
.../java/com/epam/datalab/dto/SchedulerJobDTO.java | 86 ++
.../java/com/epam/datalab/dto/StatusBaseDTO.java | 127 ++
.../com/epam/datalab/dto/StatusEnvBaseDTO.java | 117 ++
.../epam/datalab/dto/UserEnvironmentResources.java | 62 +
.../java/com/epam/datalab/dto/UserInstanceDTO.java | 192 +++
.../com/epam/datalab/dto/UserInstanceStatus.java | 67 +
.../com/epam/datalab/dto/aws/AwsCloudSettings.java | 102 ++
.../computational/AwsComputationalResource.java | 78 ++
.../AwsComputationalTerminateDTO.java | 33 +
.../dto/aws/computational/ClusterConfig.java | 41 +
.../aws/computational/ComputationalConfigAws.java | 53 +
.../aws/computational/ComputationalCreateAws.java | 165 +++
.../computational/SparkComputationalConfigAws.java | 48 +
.../computational/SparkComputationalCreateAws.java | 78 ++
.../epam/datalab/dto/aws/edge/EdgeCreateAws.java | 53 +
.../com/epam/datalab/dto/aws/edge/EdgeInfoAws.java | 42 +
.../dto/aws/exploratory/ExploratoryCreateAws.java | 48 +
.../datalab/dto/aws/keyload/UploadFileAws.java | 35 +
.../epam/datalab/dto/azure/AzureCloudSettings.java | 96 ++
.../azure/auth/AuthorizationCodeFlowResponse.java | 63 +
.../SparkComputationalConfigAzure.java | 46 +
.../SparkComputationalCreateAzure.java | 94 ++
.../datalab/dto/azure/edge/EdgeCreateAzure.java | 54 +
.../epam/datalab/dto/azure/edge/EdgeInfoAzure.java | 52 +
.../exploratory/ExploratoryActionStartAzure.java | 64 +
.../exploratory/ExploratoryActionStopAzure.java | 44 +
.../azure/exploratory/ExploratoryCreateAzure.java | 85 ++
.../datalab/dto/azure/keyload/UploadFileAzure.java | 35 +
.../com/epam/datalab/dto/backup/EnvBackupDTO.java | 40 +
.../epam/datalab/dto/backup/EnvBackupStatus.java | 49 +
.../datalab/dto/backup/EnvBackupStatusDTO.java | 56 +
.../com/epam/datalab/dto/base/CloudSettings.java | 32 +
.../com/epam/datalab/dto/base/DataEngineType.java | 66 +
.../dto/base/computational/ComputationalBase.java | 119 ++
.../computational/FullComputationalTemplate.java | 33 +
.../com/epam/datalab/dto/base/edge/EdgeInfo.java | 71 +
.../datalab/dto/base/keyload/ReuploadFile.java | 33 +
.../epam/datalab/dto/base/keyload/UploadFile.java | 33 +
.../datalab/dto/base/keyload/UploadFileResult.java | 46 +
.../datalab/dto/base/project/ProjectResult.java | 37 +
.../com/epam/datalab/dto/billing/BillingData.java | 44 +
.../datalab/dto/billing/BillingResourceType.java | 31 +
.../com/epam/datalab/dto/bucket/BucketDTO.java | 34 +
.../epam/datalab/dto/bucket/BucketDeleteDTO.java | 32 +
.../epam/datalab/dto/bucket/FolderUploadDTO.java | 33 +
.../computational/CheckInactivityCallbackDTO.java | 57 +
.../dto/computational/CheckInactivityStatus.java | 49 +
.../computational/CheckInactivityStatusDTO.java | 30 +
.../ComputationalCheckInactivityDTO.java | 57 +
.../ComputationalClusterConfigDTO.java | 40 +
.../dto/computational/ComputationalStartDTO.java | 25 +
.../dto/computational/ComputationalStatusDTO.java | 118 ++
.../dto/computational/ComputationalStopDTO.java | 25 +
.../computational/ComputationalTerminateDTO.java | 25 +
.../SparkStandaloneClusterResource.java | 62 +
.../computational/UserComputationalResource.java | 99 ++
.../dto/exploratory/ExploratoryActionDTO.java | 71 +
.../dto/exploratory/ExploratoryBaseDTO.java | 85 ++
.../ExploratoryCheckInactivityAction.java | 23 +
.../dto/exploratory/ExploratoryCreateDTO.java | 134 ++
.../dto/exploratory/ExploratoryGitCreds.java | 193 +++
.../dto/exploratory/ExploratoryGitCredsDTO.java | 63 +
.../exploratory/ExploratoryGitCredsUpdateDTO.java | 62 +
.../dto/exploratory/ExploratoryImageDTO.java | 72 ++
...xploratoryReconfigureSparkClusterActionDTO.java | 43 +
.../dto/exploratory/ExploratoryStatusDTO.java | 128 ++
.../dto/exploratory/ImageCreateStatusDTO.java | 74 ++
.../epam/datalab/dto/exploratory/ImageStatus.java | 39 +
.../datalab/dto/exploratory/LibInstallDTO.java | 85 ++
.../dto/exploratory/LibInstallStatusDTO.java | 57 +
.../datalab/dto/exploratory/LibListStatusDTO.java | 63 +
.../epam/datalab/dto/exploratory/LibStatus.java | 49 +
.../datalab/dto/exploratory/LibraryInstallDTO.java | 67 +
.../com/epam/datalab/dto/gcp/GcpCloudSettings.java | 96 ++
.../auth/GcpOauth2AuthorizationCodeResponse.java | 29 +
.../gcp/computational/ComputationalCreateGcp.java | 100 ++
.../computational/GcpComputationalResource.java | 77 ++
.../GcpComputationalTerminateDTO.java | 33 +
.../computational/SparkComputationalCreateGcp.java | 81 ++
.../epam/datalab/dto/gcp/edge/EdgeCreateGcp.java | 25 +
.../com/epam/datalab/dto/gcp/edge/EdgeInfoGcp.java | 40 +
.../dto/gcp/exploratory/ExploratoryCreateGcp.java | 48 +
.../datalab/dto/gcp/keyload/UploadFileGcp.java | 42 +
.../datalab/dto/imagemetadata/ApplicationDto.java | 85 ++
.../imagemetadata/ComputationalMetadataDTO.java | 53 +
.../ComputationalResourceShapeDto.java | 114 ++
.../ExploratoryEnvironmentVersion.java | 94 ++
.../dto/imagemetadata/ExploratoryMetadataDTO.java | 48 +
.../dto/imagemetadata/ImageMetadataDTO.java | 51 +
.../epam/datalab/dto/imagemetadata/ImageType.java | 46 +
.../datalab/dto/imagemetadata/TemplateDTO.java | 84 ++
.../epam/datalab/dto/keyload/KeyLoadStatus.java | 60 +
.../com/epam/datalab/dto/keyload/UserKeyDTO.java | 58 +
.../epam/datalab/dto/project/ProjectActionDTO.java | 34 +
.../epam/datalab/dto/project/ProjectCreateDTO.java | 37 +
.../dto/reuploadkey/ReuploadKeyCallbackDTO.java | 64 +
.../datalab/dto/reuploadkey/ReuploadKeyDTO.java | 56 +
.../datalab/dto/reuploadkey/ReuploadKeyStatus.java | 49 +
.../dto/reuploadkey/ReuploadKeyStatusDTO.java | 48 +
.../com/epam/datalab/dto/status/EnvResource.java | 161 +++
.../epam/datalab/dto/status/EnvResourceList.java | 91 ++
.../com/epam/datalab/dto/status/EnvStatusDTO.java | 65 +
.../java/com/epam/datalab/model/ResourceData.java | 60 +
.../java/com/epam/datalab/model/ResourceEnum.java | 38 +
.../java/com/epam/datalab/model/ResourceType.java | 37 +
.../java/com/epam/datalab/model/StringList.java | 47 +
.../datalab/model/aws/BillingResourceType.java | 80 ++
.../com/epam/datalab/model/aws/ReportLine.java | 233 ++++
.../epam/datalab/model/azure/AzureAuthFile.java | 39 +
.../datalab/model/exploratory/Exploratory.java | 41 +
.../com/epam/datalab/model/exploratory/Image.java | 46 +
.../com/epam/datalab/model/library/Library.java | 55 +
.../datalab/model/scheduler/SchedulerJobData.java | 44 +
.../epam/datalab/model/systeminfo/DiskInfo.java | 36 +
.../epam/datalab/model/systeminfo/MemoryInfo.java | 39 +
.../com/epam/datalab/model/systeminfo/OsInfo.java | 38 +
.../datalab/model/systeminfo/ProcessorInfo.java | 46 +
.../datalab/util/CloudSettingsDeserializer.java | 88 ++
.../java/com/epam/datalab/util/JsonGenerator.java | 54 +
.../datalab/dto/status/EnvResourceDTOTest.java | 80 ++
.../epam/datalab/dto/status/EnvStatusDTOTest.java | 89 ++
.../com/epam/datalab/util/JsonGeneratorTest.java | 37 +
services/datalab-mongo-migration/pom.xml | 39 +
.../com/epam/datalab/migration/DbMigration.java | 26 +
.../exception/DatalabDbMigrationException.java | 27 +
.../migration/mongo/DatalabMongoMigration.java | 48 +
.../mongo/changelog/DatalabChangeLog.java | 66 +
services/datalab-utils/pom.xml | 52 +
.../main/java/com/epam/datalab/util/FileUtils.java | 61 +
.../java/com/epam/datalab/util/SecurityUtils.java | 38 +
.../java/com/epam/datalab/util/ServiceUtils.java | 151 +++
.../java/com/epam/datalab/util/UsernameUtils.java | 36 +
.../datalab/util/mongo/IsoDateDeSerializer.java | 61 +
.../epam/datalab/util/mongo/IsoDateSerializer.java | 48 +
.../util/mongo/IsoLocalDateDeSerializer.java | 47 +
.../datalab/util/mongo/IsoLocalDateSerializer.java | 38 +
.../util/mongo/IsoLocalDateTimeDeSerializer.java | 48 +
.../util/mongo/IsoLocalDateTimeSerializer.java | 38 +
.../epam/datalab/util/mongo/LongDeSerializer.java | 44 +
.../util/mongo/MongoStringDeserializer.java | 32 +
.../datalab/util/mongo/MongoStringSerializaer.java | 38 +
.../datalab/util/mongo/modules/IsoDateModule.java | 64 +
.../util/mongo/modules/JavaPrimitiveModule.java | 30 +
.../datalab/util/mongo/modules/MongoModule.java | 31 +
.../com/epam/datalab/util/SecurityUtilsTest.java | 35 +
.../mongo/IsoLocalDateTimeDeSerializerTest.java | 89 ++
.../util/mongo/IsoLocalDateTimeSerDeTest.java | 70 +
.../util/mongo/IsoLocalDateTimeSerializerTest.java | 57 +
services/datalab-webapp-common/pom.xml | 69 +
.../src/main/java/com/epam/datalab/ModuleBase.java | 48 +
.../com/epam/datalab/ServiceConfiguration.java | 114 ++
.../datalab/auth/SecurityUnauthorizedHandler.java | 32 +
.../main/java/com/epam/datalab/auth/UserInfo.java | 222 ++++
.../epam/datalab/auth/contract/SecurityAPI.java | 30 +
.../epam/datalab/auth/dto/UserCredentialDTO.java | 74 ++
.../auth/oauth2/Oauth2AuthenticationService.java | 38 +
.../auth/rest/AbstractAuthenticationService.java | 47 +
.../datalab/auth/rest/ConfigurableResource.java | 36 +
.../auth/rest/UserSessionDurationAuthorizer.java | 90 ++
.../auth/rest/UserSessionDurationCallback.java | 27 +
.../java/com/epam/datalab/cloud/CloudModule.java | 34 +
.../java/com/epam/datalab/cloud/CloudProvider.java | 28 +
.../com/epam/datalab/constants/ServiceConsts.java | 34 +
.../java/com/epam/datalab/mongo/MongoService.java | 80 ++
.../epam/datalab/mongo/MongoServiceFactory.java | 93 ++
.../com/epam/datalab/rest/client/RESTService.java | 165 +++
.../datalab/rest/client/RESTServiceFactory.java | 61 +
.../epam/datalab/rest/contracts/ApiCallbacks.java | 45 +
.../com/epam/datalab/rest/contracts/BackupAPI.java | 27 +
.../datalab/rest/contracts/ComputationalAPI.java | 41 +
.../com/epam/datalab/rest/contracts/DockerAPI.java | 27 +
.../datalab/rest/contracts/ExploratoryAPI.java | 34 +
.../datalab/rest/contracts/InfrasctructureAPI.java | 29 +
.../com/epam/datalab/rest/contracts/KeyAPI.java | 29 +
.../java/com/epam/datalab/rest/dto/ErrorDTO.java | 35 +
.../mappers/AuthenticationExceptionMapper.java | 38 +
.../mappers/DatalabValidationExceptionMapper.java | 38 +
.../rest/mappers/GenericExceptionMapper.java | 43 +
.../mappers/JsonProcessingExceptionMapper.java | 28 +
.../mappers/ResourceConflictExceptionMapper.java | 40 +
.../mappers/ResourceNotFoundExceptionMapper.java | 40 +
.../ResourceQuoteReachedExceptionMapper.java | 37 +
.../rest/mappers/RuntimeExceptionMapper.java | 63 +
.../rest/mappers/ValidationExceptionMapper.java | 45 +
.../com/epam/datalab/validation/AwsValidation.java | 23 +
.../epam/datalab/validation/AzureValidation.java | 23 +
.../CloudConfigurationSequenceProvider.java | 66 +
.../com/epam/datalab/validation/GcpValidation.java | 23 +
services/dlab-model/pom.xml | 80 --
.../src/main/java/com/epam/dlab/MongoKeyWords.java | 34 -
.../epam/dlab/dto/InfrastructureMetaInfoDTO.java | 34 -
.../com/epam/dlab/dto/LibListComputationalDTO.java | 55 -
.../com/epam/dlab/dto/LibListExploratoryDTO.java | 40 -
.../main/java/com/epam/dlab/dto/LibraryGroups.java | 40 -
.../java/com/epam/dlab/dto/ResourceBaseDTO.java | 74 --
.../java/com/epam/dlab/dto/ResourceEnvBaseDTO.java | 71 -
.../java/com/epam/dlab/dto/ResourceSysBaseDTO.java | 87 --
.../main/java/com/epam/dlab/dto/ResourceURL.java | 38 -
.../java/com/epam/dlab/dto/SchedulerJobDTO.java | 82 --
.../main/java/com/epam/dlab/dto/StatusBaseDTO.java | 127 --
.../java/com/epam/dlab/dto/StatusEnvBaseDTO.java | 117 --
.../epam/dlab/dto/UserEnvironmentResources.java | 62 -
.../java/com/epam/dlab/dto/UserInstanceDTO.java | 192 ---
.../java/com/epam/dlab/dto/UserInstanceStatus.java | 67 -
.../com/epam/dlab/dto/aws/AwsCloudSettings.java | 97 --
.../computational/AwsComputationalResource.java | 78 --
.../AwsComputationalTerminateDTO.java | 33 -
.../dlab/dto/aws/computational/ClusterConfig.java | 41 -
.../aws/computational/ComputationalConfigAws.java | 53 -
.../aws/computational/ComputationalCreateAws.java | 165 ---
.../computational/SparkComputationalConfigAws.java | 48 -
.../computational/SparkComputationalCreateAws.java | 78 --
.../com/epam/dlab/dto/aws/edge/EdgeCreateAws.java | 53 -
.../com/epam/dlab/dto/aws/edge/EdgeInfoAws.java | 42 -
.../dto/aws/exploratory/ExploratoryCreateAws.java | 48 -
.../epam/dlab/dto/aws/keyload/UploadFileAws.java | 35 -
.../epam/dlab/dto/azure/AzureCloudSettings.java | 91 --
.../azure/auth/AuthorizationCodeFlowResponse.java | 63 -
.../SparkComputationalConfigAzure.java | 46 -
.../SparkComputationalCreateAzure.java | 94 --
.../epam/dlab/dto/azure/edge/EdgeCreateAzure.java | 54 -
.../epam/dlab/dto/azure/edge/EdgeInfoAzure.java | 52 -
.../exploratory/ExploratoryActionStartAzure.java | 64 -
.../exploratory/ExploratoryActionStopAzure.java | 44 -
.../azure/exploratory/ExploratoryCreateAzure.java | 85 --
.../dlab/dto/azure/keyload/UploadFileAzure.java | 35 -
.../com/epam/dlab/dto/backup/EnvBackupDTO.java | 40 -
.../com/epam/dlab/dto/backup/EnvBackupStatus.java | 49 -
.../epam/dlab/dto/backup/EnvBackupStatusDTO.java | 56 -
.../java/com/epam/dlab/dto/base/CloudSettings.java | 32 -
.../com/epam/dlab/dto/base/DataEngineType.java | 66 -
.../dto/base/computational/ComputationalBase.java | 119 --
.../computational/FullComputationalTemplate.java | 33 -
.../java/com/epam/dlab/dto/base/edge/EdgeInfo.java | 71 -
.../epam/dlab/dto/base/keyload/ReuploadFile.java | 33 -
.../com/epam/dlab/dto/base/keyload/UploadFile.java | 33 -
.../dlab/dto/base/keyload/UploadFileResult.java | 46 -
.../epam/dlab/dto/base/project/ProjectResult.java | 37 -
.../com/epam/dlab/dto/billing/BillingData.java | 44 -
.../epam/dlab/dto/billing/BillingResourceType.java | 31 -
.../java/com/epam/dlab/dto/bucket/BucketDTO.java | 34 -
.../com/epam/dlab/dto/bucket/BucketDeleteDTO.java | 32 -
.../com/epam/dlab/dto/bucket/FolderUploadDTO.java | 33 -
.../computational/CheckInactivityCallbackDTO.java | 57 -
.../dto/computational/CheckInactivityStatus.java | 49 -
.../computational/CheckInactivityStatusDTO.java | 30 -
.../ComputationalCheckInactivityDTO.java | 57 -
.../ComputationalClusterConfigDTO.java | 40 -
.../dto/computational/ComputationalStartDTO.java | 25 -
.../dto/computational/ComputationalStatusDTO.java | 118 --
.../dto/computational/ComputationalStopDTO.java | 25 -
.../computational/ComputationalTerminateDTO.java | 25 -
.../SparkStandaloneClusterResource.java | 62 -
.../computational/UserComputationalResource.java | 99 --
.../dlab/dto/exploratory/ExploratoryActionDTO.java | 71 -
.../dlab/dto/exploratory/ExploratoryBaseDTO.java | 84 --
.../ExploratoryCheckInactivityAction.java | 23 -
.../dlab/dto/exploratory/ExploratoryCreateDTO.java | 134 --
.../dlab/dto/exploratory/ExploratoryGitCreds.java | 193 ---
.../dto/exploratory/ExploratoryGitCredsDTO.java | 60 -
.../exploratory/ExploratoryGitCredsUpdateDTO.java | 58 -
.../dlab/dto/exploratory/ExploratoryImageDTO.java | 72 --
...xploratoryReconfigureSparkClusterActionDTO.java | 43 -
.../dlab/dto/exploratory/ExploratoryStatusDTO.java | 128 --
.../dlab/dto/exploratory/ImageCreateStatusDTO.java | 74 --
.../com/epam/dlab/dto/exploratory/ImageStatus.java | 39 -
.../epam/dlab/dto/exploratory/LibInstallDTO.java | 85 --
.../dlab/dto/exploratory/LibInstallStatusDTO.java | 57 -
.../dlab/dto/exploratory/LibListStatusDTO.java | 63 -
.../com/epam/dlab/dto/exploratory/LibStatus.java | 49 -
.../dlab/dto/exploratory/LibraryInstallDTO.java | 67 -
.../com/epam/dlab/dto/gcp/GcpCloudSettings.java | 91 --
.../auth/GcpOauth2AuthorizationCodeResponse.java | 29 -
.../gcp/computational/ComputationalCreateGcp.java | 100 --
.../computational/GcpComputationalResource.java | 77 --
.../GcpComputationalTerminateDTO.java | 33 -
.../computational/SparkComputationalCreateGcp.java | 81 --
.../com/epam/dlab/dto/gcp/edge/EdgeCreateGcp.java | 25 -
.../com/epam/dlab/dto/gcp/edge/EdgeInfoGcp.java | 40 -
.../dto/gcp/exploratory/ExploratoryCreateGcp.java | 48 -
.../epam/dlab/dto/gcp/keyload/UploadFileGcp.java | 42 -
.../dlab/dto/imagemetadata/ApplicationDto.java | 85 --
.../imagemetadata/ComputationalMetadataDTO.java | 53 -
.../ComputationalResourceShapeDto.java | 114 --
.../ExploratoryEnvironmentVersion.java | 94 --
.../dto/imagemetadata/ExploratoryMetadataDTO.java | 48 -
.../dlab/dto/imagemetadata/ImageMetadataDTO.java | 51 -
.../com/epam/dlab/dto/imagemetadata/ImageType.java | 46 -
.../epam/dlab/dto/imagemetadata/TemplateDTO.java | 84 --
.../com/epam/dlab/dto/keyload/KeyLoadStatus.java | 60 -
.../java/com/epam/dlab/dto/keyload/UserKeyDTO.java | 58 -
.../epam/dlab/dto/project/ProjectActionDTO.java | 34 -
.../epam/dlab/dto/project/ProjectCreateDTO.java | 37 -
.../dto/reuploadkey/ReuploadKeyCallbackDTO.java | 64 -
.../epam/dlab/dto/reuploadkey/ReuploadKeyDTO.java | 56 -
.../dlab/dto/reuploadkey/ReuploadKeyStatus.java | 49 -
.../dlab/dto/reuploadkey/ReuploadKeyStatusDTO.java | 48 -
.../java/com/epam/dlab/dto/status/EnvResource.java | 161 ---
.../com/epam/dlab/dto/status/EnvResourceList.java | 78 --
.../com/epam/dlab/dto/status/EnvStatusDTO.java | 58 -
.../java/com/epam/dlab/model/ResourceData.java | 60 -
.../java/com/epam/dlab/model/ResourceEnum.java | 38 -
.../java/com/epam/dlab/model/ResourceType.java | 37 -
.../main/java/com/epam/dlab/model/StringList.java | 47 -
.../epam/dlab/model/aws/BillingResourceType.java | 75 --
.../java/com/epam/dlab/model/aws/ReportLine.java | 226 ----
.../com/epam/dlab/model/azure/AzureAuthFile.java | 39 -
.../epam/dlab/model/exploratory/Exploratory.java | 41 -
.../com/epam/dlab/model/exploratory/Image.java | 46 -
.../java/com/epam/dlab/model/library/Library.java | 55 -
.../dlab/model/scheduler/SchedulerJobData.java | 44 -
.../com/epam/dlab/model/systeminfo/DiskInfo.java | 36 -
.../com/epam/dlab/model/systeminfo/MemoryInfo.java | 39 -
.../com/epam/dlab/model/systeminfo/OsInfo.java | 38 -
.../epam/dlab/model/systeminfo/ProcessorInfo.java | 46 -
.../epam/dlab/util/CloudSettingsDeserializer.java | 88 --
.../java/com/epam/dlab/util/JsonGenerator.java | 54 -
.../epam/dlab/dto/status/EnvResourceDTOTest.java | 81 --
.../com/epam/dlab/dto/status/EnvStatusDTOTest.java | 89 --
.../java/com/epam/dlab/util/JsonGeneratorTest.java | 37 -
services/dlab-mongo-migration/pom.xml | 39 -
.../java/com/epam/dlab/migration/DbMigration.java | 26 -
.../exception/DlabDbMigrationException.java | 27 -
.../dlab/migration/mongo/DlabMongoMigration.java | 48 -
.../migration/mongo/changelog/DlabChangeLog.java | 66 -
services/dlab-utils/pom.xml | 52 -
.../main/java/com/epam/dlab/util/FileUtils.java | 61 -
.../java/com/epam/dlab/util/SecurityUtils.java | 38 -
.../main/java/com/epam/dlab/util/ServiceUtils.java | 141 --
.../java/com/epam/dlab/util/UsernameUtils.java | 36 -
.../epam/dlab/util/mongo/IsoDateDeSerializer.java | 61 -
.../epam/dlab/util/mongo/IsoDateSerializer.java | 48 -
.../dlab/util/mongo/IsoLocalDateDeSerializer.java | 47 -
.../dlab/util/mongo/IsoLocalDateSerializer.java | 38 -
.../util/mongo/IsoLocalDateTimeDeSerializer.java | 48 -
.../util/mongo/IsoLocalDateTimeSerializer.java | 38 -
.../com/epam/dlab/util/mongo/LongDeSerializer.java | 44 -
.../dlab/util/mongo/MongoStringDeserializer.java | 32 -
.../dlab/util/mongo/MongoStringSerializaer.java | 38 -
.../dlab/util/mongo/modules/IsoDateModule.java | 59 -
.../util/mongo/modules/JavaPrimitiveModule.java | 30 -
.../epam/dlab/util/mongo/modules/MongoModule.java | 31 -
.../java/com/epam/dlab/util/SecurityUtilsTest.java | 35 -
.../mongo/IsoLocalDateTimeDeSerializerTest.java | 84 --
.../dlab/util/mongo/IsoLocalDateTimeSerDeTest.java | 70 -
.../util/mongo/IsoLocalDateTimeSerializerTest.java | 54 -
services/dlab-webapp-common/pom.xml | 69 -
.../src/main/java/com/epam/dlab/ModuleBase.java | 42 -
.../java/com/epam/dlab/ServiceConfiguration.java | 114 --
.../dlab/auth/SecurityUnauthorizedHandler.java | 32 -
.../src/main/java/com/epam/dlab/auth/UserInfo.java | 223 ----
.../com/epam/dlab/auth/contract/SecurityAPI.java | 30 -
.../com/epam/dlab/auth/dto/UserCredentialDTO.java | 74 --
.../auth/oauth2/Oauth2AuthenticationService.java | 38 -
.../auth/rest/AbstractAuthenticationService.java | 47 -
.../epam/dlab/auth/rest/ConfigurableResource.java | 36 -
.../auth/rest/UserSessionDurationAuthorizer.java | 90 --
.../auth/rest/UserSessionDurationCallback.java | 27 -
.../main/java/com/epam/dlab/cloud/CloudModule.java | 34 -
.../java/com/epam/dlab/cloud/CloudProvider.java | 28 -
.../com/epam/dlab/constants/ServiceConsts.java | 34 -
.../java/com/epam/dlab/mongo/MongoService.java | 80 --
.../com/epam/dlab/mongo/MongoServiceFactory.java | 93 --
.../com/epam/dlab/rest/client/RESTService.java | 165 ---
.../epam/dlab/rest/client/RESTServiceFactory.java | 61 -
.../com/epam/dlab/rest/contracts/ApiCallbacks.java | 45 -
.../com/epam/dlab/rest/contracts/BackupAPI.java | 26 -
.../epam/dlab/rest/contracts/ComputationalAPI.java | 41 -
.../com/epam/dlab/rest/contracts/DockerAPI.java | 27 -
.../epam/dlab/rest/contracts/ExploratoryAPI.java | 34 -
.../dlab/rest/contracts/InfrasctructureAPI.java | 29 -
.../java/com/epam/dlab/rest/contracts/KeyAPI.java | 29 -
.../main/java/com/epam/dlab/rest/dto/ErrorDTO.java | 35 -
.../mappers/AuthenticationExceptionMapper.java | 38 -
.../mappers/DlabValidationExceptionMapper.java | 38 -
.../dlab/rest/mappers/GenericExceptionMapper.java | 43 -
.../mappers/JsonProcessingExceptionMapper.java | 28 -
.../mappers/ResourceConflictExceptionMapper.java | 40 -
.../mappers/ResourceNotFoundExceptionMapper.java | 40 -
.../ResourceQuoteReachedExceptionMapper.java | 37 -
.../dlab/rest/mappers/RuntimeExceptionMapper.java | 63 -
.../rest/mappers/ValidationExceptionMapper.java | 45 -
.../com/epam/dlab/validation/AwsValidation.java | 23 -
.../com/epam/dlab/validation/AzureValidation.java | 23 -
.../CloudConfigurationSequenceProvider.java | 66 -
.../com/epam/dlab/validation/GcpValidation.java | 23 -
services/provisioning-service/pom.xml | 32 +-
services/provisioning-service/provisioning.yml | 14 +-
.../DropwizardBearerTokenFilterImpl.java | 39 +
.../backendapi/ProvisioningServiceApplication.java | 161 +++
...rovisioningServiceApplicationConfiguration.java | 197 +++
.../backendapi/conf/CloudConfiguration.java | 83 ++
.../epam/datalab/backendapi/core/Constants.java | 29 +
.../epam/datalab/backendapi/core/Directories.java | 31 +
.../backendapi/core/DirectoriesCreator.java | 49 +
.../datalab/backendapi/core/DockerWarmuper.java | 172 +++
.../backendapi/core/FileHandlerCallback.java | 41 +
.../datalab/backendapi/core/MetadataHolder.java | 31 +
.../backendapi/core/commands/CmdCommand.java | 25 +
.../backendapi/core/commands/CommandBuilder.java | 186 +++
.../backendapi/core/commands/CommandExecutor.java | 38 +
.../core/commands/CommandExecutorMock.java | 114 ++
.../core/commands/CommandExecutorMockAsync.java | 406 ++++++
.../core/commands/CommandParserMock.java | 377 ++++++
.../backendapi/core/commands/DockerAction.java | 54 +
.../backendapi/core/commands/DockerCommands.java | 69 +
.../backendapi/core/commands/ICommandExecutor.java | 27 +
.../core/commands/ImagesDockerCommand.java | 53 +
.../core/commands/PythonBackupCommand.java | 81 ++
.../backendapi/core/commands/PythonCommand.java | 50 +
.../backendapi/core/commands/RunDockerCommand.java | 285 ++++
.../backendapi/core/commands/UnixCommand.java | 53 +
.../response/folderlistener/AsyncFileHandler.java | 135 ++
.../response/folderlistener/FolderListener.java | 434 +++++++
.../folderlistener/FolderListenerExecutor.java | 55 +
.../core/response/folderlistener/WatchItem.java | 258 ++++
.../response/folderlistener/WatchItemList.java | 277 ++++
.../response/handlers/BackupCallbackHandler.java | 126 ++
.../handlers/CheckInactivityCallbackHandler.java | 141 ++
.../handlers/ComputationalCallbackHandler.java | 133 ++
.../response/handlers/ComputationalConfigure.java | 140 ++
.../ComputationalConfigureCallbackHandler.java | 63 +
.../response/handlers/EdgeCallbackHandler.java | 78 ++
.../handlers/ExploratoryCallbackHandler.java | 108 ++
.../ExploratoryGitCredsCallbackHandler.java | 65 +
.../handlers/ImageCreateCallbackHandler.java | 106 ++
.../handlers/LibInstallCallbackHandler.java | 118 ++
.../response/handlers/LibListCallbackHandler.java | 110 ++
.../response/handlers/PersistentFileHandler.java | 51 +
.../response/handlers/ProjectCallbackHandler.java | 71 +
.../response/handlers/ResourceCallbackHandler.java | 210 +++
.../handlers/ResourcesStatusCallbackHandler.java | 93 ++
.../handlers/ReuploadKeyCallbackHandler.java | 124 ++
.../response/handlers/dao/CallbackHandlerDao.java | 33 +
.../handlers/dao/FileSystemCallbackHandlerDao.java | 121 ++
.../backendapi/modules/AwsProvisioningModule.java | 39 +
.../modules/AzureProvisioningModule.java | 39 +
.../modules/CloudModuleConfigurator.java | 42 +
.../backendapi/modules/GcpProvisioningModule.java | 39 +
.../datalab/backendapi/modules/ModuleFactory.java | 44 +
.../backendapi/modules/ProductionModule.java | 89 ++
.../backendapi/modules/ProvisioningDevModule.java | 148 +++
.../backendapi/resources/BackupResource.java | 76 ++
.../backendapi/resources/BucketResource.java | 144 +++
.../resources/CallbackHandlerResource.java | 46 +
.../backendapi/resources/DockerResource.java | 92 ++
.../resources/GitExploratoryResource.java | 95 ++
.../backendapi/resources/ImageResource.java | 88 ++
.../resources/InfrastructureResource.java | 65 +
.../backendapi/resources/LibraryResource.java | 195 +++
.../backendapi/resources/ProjectResource.java | 76 ++
.../resources/ProvisioningHealthCheckResource.java | 44 +
.../resources/aws/ComputationalResourceAws.java | 197 +++
.../backendapi/resources/aws/EdgeResourceAws.java | 94 ++
.../resources/aws/ExploratoryResourceAws.java | 77 ++
.../resources/aws/InfrastructureResourceAws.java | 49 +
.../azure/ComputationalResourceAzure.java | 94 ++
.../resources/azure/EdgeResourceAzure.java | 94 ++
.../resources/azure/ExploratoryResourceAzure.java | 75 ++
.../azure/InfrastructureResourceAzure.java | 50 +
.../backendapi/resources/base/EdgeService.java | 108 ++
.../resources/base/ExploratoryService.java | 79 ++
.../resources/base/InfrastructureService.java | 157 +++
.../backendapi/resources/base/KeyResource.java | 86 ++
.../resources/gcp/ComputationalResourceGcp.java | 197 +++
.../backendapi/resources/gcp/EdgeResourceGcp.java | 96 ++
.../resources/gcp/ExploratoryResourceGcp.java | 77 ++
.../resources/gcp/InfrastructureResourceGcp.java | 49 +
.../datalab/backendapi/service/BucketService.java | 40 +
.../backendapi/service/CheckInactivityService.java | 29 +
.../datalab/backendapi/service/ProjectService.java | 35 +
.../service/RestoreCallbackHandlerService.java | 25 +
.../service/impl/CheckInactivityServiceImpl.java | 129 ++
.../backendapi/service/impl/DockerService.java | 42 +
.../backendapi/service/impl/KeyService.java | 146 +++
.../service/impl/ProjectServiceImpl.java | 135 ++
.../impl/RestoreCallbackHandlerServiceImpl.java | 58 +
.../service/impl/SparkClusterService.java | 156 +++
.../service/impl/aws/BucketServiceAwsImpl.java | 149 +++
.../service/impl/azure/BucketServiceAzureImpl.java | 176 +++
.../service/impl/gcp/BucketServiceGcpImpl.java | 128 ++
...gServiceCloudConfigurationSequenceProvider.java | 28 +
.../com/epam/datalab/process/ProcessConveyor.java | 59 +
.../process/builder/ProcessInfoBuilder.java | 288 +++++
.../process/exception/DatalabProcessException.java | 42 +
.../epam/datalab/process/model/DatalabProcess.java | 167 +++
.../com/epam/datalab/process/model/ProcessId.java | 70 +
.../epam/datalab/process/model/ProcessInfo.java | 120 ++
.../epam/datalab/process/model/ProcessStatus.java | 32 +
.../epam/datalab/process/model/ProcessStep.java | 47 +
.../DropwizardBearerTokenFilterImpl.java | 39 -
.../backendapi/ProvisioningServiceApplication.java | 161 ---
...rovisioningServiceApplicationConfiguration.java | 197 ---
.../dlab/backendapi/conf/CloudConfiguration.java | 83 --
.../com/epam/dlab/backendapi/core/Constants.java | 29 -
.../com/epam/dlab/backendapi/core/Directories.java | 31 -
.../dlab/backendapi/core/DirectoriesCreator.java | 49 -
.../epam/dlab/backendapi/core/DockerWarmuper.java | 167 ---
.../dlab/backendapi/core/FileHandlerCallback.java | 41 -
.../epam/dlab/backendapi/core/MetadataHolder.java | 31 -
.../dlab/backendapi/core/commands/CmdCommand.java | 25 -
.../backendapi/core/commands/CommandBuilder.java | 186 ---
.../backendapi/core/commands/CommandExecutor.java | 38 -
.../core/commands/CommandExecutorMock.java | 114 --
.../core/commands/CommandExecutorMockAsync.java | 406 ------
.../core/commands/CommandParserMock.java | 373 ------
.../backendapi/core/commands/DockerAction.java | 54 -
.../backendapi/core/commands/DockerCommands.java | 69 -
.../backendapi/core/commands/ICommandExecutor.java | 26 -
.../core/commands/ImagesDockerCommand.java | 53 -
.../core/commands/PythonBackupCommand.java | 81 --
.../backendapi/core/commands/PythonCommand.java | 50 -
.../backendapi/core/commands/RunDockerCommand.java | 285 ----
.../dlab/backendapi/core/commands/UnixCommand.java | 53 -
.../response/folderlistener/AsyncFileHandler.java | 135 --
.../response/folderlistener/FolderListener.java | 434 -------
.../folderlistener/FolderListenerExecutor.java | 55 -
.../core/response/folderlistener/WatchItem.java | 214 ---
.../response/folderlistener/WatchItemList.java | 277 ----
.../response/handlers/BackupCallbackHandler.java | 126 --
.../handlers/CheckInactivityCallbackHandler.java | 141 --
.../handlers/ComputationalCallbackHandler.java | 133 --
.../response/handlers/ComputationalConfigure.java | 140 --
.../ComputationalConfigureCallbackHandler.java | 63 -
.../response/handlers/EdgeCallbackHandler.java | 78 --
.../handlers/ExploratoryCallbackHandler.java | 108 --
.../ExploratoryGitCredsCallbackHandler.java | 65 -
.../handlers/ImageCreateCallbackHandler.java | 106 --
.../handlers/LibInstallCallbackHandler.java | 118 --
.../response/handlers/LibListCallbackHandler.java | 110 --
.../response/handlers/PersistentFileHandler.java | 51 -
.../response/handlers/ProjectCallbackHandler.java | 71 -
.../response/handlers/ResourceCallbackHandler.java | 210 ---
.../handlers/ResourcesStatusCallbackHandler.java | 93 --
.../handlers/ReuploadKeyCallbackHandler.java | 124 --
.../response/handlers/dao/CallbackHandlerDao.java | 33 -
.../handlers/dao/FileSystemCallbackHandlerDao.java | 121 --
.../backendapi/modules/AwsProvisioningModule.java | 39 -
.../modules/AzureProvisioningModule.java | 39 -
.../modules/CloudModuleConfigurator.java | 42 -
.../backendapi/modules/GcpProvisioningModule.java | 39 -
.../dlab/backendapi/modules/ModuleFactory.java | 44 -
.../dlab/backendapi/modules/ProductionModule.java | 89 --
.../backendapi/modules/ProvisioningDevModule.java | 148 ---
.../dlab/backendapi/resources/BackupResource.java | 76 --
.../dlab/backendapi/resources/BucketResource.java | 144 ---
.../resources/CallbackHandlerResource.java | 46 -
.../dlab/backendapi/resources/DockerResource.java | 87 --
.../resources/GitExploratoryResource.java | 95 --
.../dlab/backendapi/resources/ImageResource.java | 88 --
.../resources/InfrastructureResource.java | 61 -
.../dlab/backendapi/resources/LibraryResource.java | 195 ---
.../dlab/backendapi/resources/ProjectResource.java | 76 --
.../resources/ProvisioningHealthCheckResource.java | 44 -
.../resources/aws/ComputationalResourceAws.java | 197 ---
.../backendapi/resources/aws/EdgeResourceAws.java | 94 --
.../resources/aws/ExploratoryResourceAws.java | 77 --
.../resources/aws/InfrastructureResourceAws.java | 49 -
.../azure/ComputationalResourceAzure.java | 94 --
.../resources/azure/EdgeResourceAzure.java | 94 --
.../resources/azure/ExploratoryResourceAzure.java | 75 --
.../azure/InfrastructureResourceAzure.java | 50 -
.../backendapi/resources/base/EdgeService.java | 104 --
.../resources/base/ExploratoryService.java | 79 --
.../resources/base/InfrastructureService.java | 157 ---
.../backendapi/resources/base/KeyResource.java | 80 --
.../resources/gcp/ComputationalResourceGcp.java | 197 ---
.../backendapi/resources/gcp/EdgeResourceGcp.java | 96 --
.../resources/gcp/ExploratoryResourceGcp.java | 77 --
.../resources/gcp/InfrastructureResourceGcp.java | 49 -
.../dlab/backendapi/service/BucketService.java | 40 -
.../backendapi/service/CheckInactivityService.java | 29 -
.../dlab/backendapi/service/ProjectService.java | 35 -
.../service/RestoreCallbackHandlerService.java | 25 -
.../service/impl/CheckInactivityServiceImpl.java | 129 --
.../backendapi/service/impl/DockerService.java | 42 -
.../dlab/backendapi/service/impl/KeyService.java | 146 ---
.../service/impl/ProjectServiceImpl.java | 135 --
.../impl/RestoreCallbackHandlerServiceImpl.java | 58 -
.../service/impl/SparkClusterService.java | 156 ---
.../service/impl/aws/BucketServiceAwsImpl.java | 149 ---
.../service/impl/azure/BucketServiceAzureImpl.java | 176 ---
.../service/impl/gcp/BucketServiceGcpImpl.java | 128 --
...gServiceCloudConfigurationSequenceProvider.java | 28 -
.../com/epam/dlab/process/ProcessConveyor.java | 59 -
.../dlab/process/builder/ProcessInfoBuilder.java | 288 -----
.../process/exception/DlabProcessException.java | 42 -
.../com/epam/dlab/process/model/DlabProcess.java | 167 ---
.../com/epam/dlab/process/model/ProcessId.java | 70 -
.../com/epam/dlab/process/model/ProcessInfo.java | 120 --
.../com/epam/dlab/process/model/ProcessStatus.java | 32 -
.../com/epam/dlab/process/model/ProcessStep.java | 47 -
.../main/resources/mock_response/aws/backup.json | 2 +-
.../aws/dataengine-service_configure.json | 2 +-
.../aws/dataengine-service_configure_failed.json | 2 +-
.../aws/dataengine-service_create.json | 2 +-
.../aws/dataengine-service_create_failed.json | 2 +-
.../aws/dataengine-service_lib_install.json | 2 +-
.../aws/dataengine-service_lib_list.json | 4 +-
.../aws/dataengine-service_terminate.json | 2 +-
.../mock_response/aws/dataengine_configure.json | 2 +-
.../mock_response/aws/dataengine_create.json | 2 +-
.../mock_response/aws/dataengine_lib_install.json | 2 +-
.../mock_response/aws/dataengine_lib_list.json | 4 +-
.../mock_response/aws/dataengine_start.json | 2 +-
.../mock_response/aws/dataengine_stop.json | 2 +-
.../mock_response/aws/dataengine_terminate.json | 2 +-
.../resources/mock_response/aws/edge_create.json | 4 +-
.../resources/mock_response/aws/edge_start.json | 2 +-
.../resources/mock_response/aws/edge_stop.json | 2 +-
.../mock_response/aws/edge_terminate.json | 2 +-
.../aws/notebook_check_inactivity.json | 2 +-
.../mock_response/aws/notebook_create.json | 2 +-
.../mock_response/aws/notebook_create_failed.json | 2 +-
.../mock_response/aws/notebook_create_image.json | 2 +-
.../mock_response/aws/notebook_git_creds.json | 2 +-
.../mock_response/aws/notebook_lib_install.json | 6 +-
.../mock_response/aws/notebook_lib_list.json | 4 +-
.../mock_response/aws/notebook_start.json | 2 +-
.../mock_response/aws/notebook_status.json | 14 +-
.../resources/mock_response/aws/notebook_stop.json | 2 +-
.../mock_response/aws/notebook_terminate.json | 2 +-
.../mock_response/aws/project_create.json | 8 +-
.../mock_response/aws/project_terminate.json | 2 +-
.../main/resources/mock_response/azure/backup.json | 2 +-
.../azure/dataengine-service_configure.json | 2 +-
.../azure/dataengine-service_configure_failed.json | 2 +-
.../azure/dataengine-service_create.json | 2 +-
.../azure/dataengine-service_create_failed.json | 2 +-
.../azure/dataengine-service_lib_install.json | 2 +-
.../azure/dataengine-service_lib_list.json | 4 +-
.../azure/dataengine-service_terminate.json | 2 +-
.../mock_response/azure/dataengine_configure.json | 2 +-
.../mock_response/azure/dataengine_create.json | 2 +-
.../azure/dataengine_lib_install.json | 2 +-
.../mock_response/azure/dataengine_lib_list.json | 4 +-
.../mock_response/azure/dataengine_start.json | 2 +-
.../mock_response/azure/dataengine_stop.json | 2 +-
.../mock_response/azure/dataengine_terminate.json | 2 +-
.../resources/mock_response/azure/edge_create.json | 2 +-
.../resources/mock_response/azure/edge_start.json | 2 +-
.../resources/mock_response/azure/edge_stop.json | 2 +-
.../mock_response/azure/edge_terminate.json | 2 +-
.../mock_response/azure/notebook_create.json | 2 +-
.../azure/notebook_create_failed.json | 2 +-
.../mock_response/azure/notebook_create_image.json | 2 +-
.../mock_response/azure/notebook_git_creds.json | 2 +-
.../mock_response/azure/notebook_lib_install.json | 6 +-
.../mock_response/azure/notebook_lib_list.json | 4 +-
.../mock_response/azure/notebook_start.json | 2 +-
.../mock_response/azure/notebook_status.json | 14 +-
.../mock_response/azure/notebook_stop.json | 2 +-
.../mock_response/azure/notebook_terminate.json | 2 +-
.../mock_response/azure/project_create.json | 8 +-
.../mock_response/azure/project_terminate.json | 2 +-
.../main/resources/mock_response/gcp/backup.json | 2 +-
.../gcp/dataengine-service_configure.json | 2 +-
.../gcp/dataengine-service_create.json | 2 +-
.../gcp/dataengine-service_lib_install.json | 2 +-
.../gcp/dataengine-service_lib_list.json | 4 +-
.../gcp/dataengine-service_terminate.json | 2 +-
.../mock_response/gcp/dataengine_configure.json | 2 +-
.../mock_response/gcp/dataengine_create.json | 2 +-
.../mock_response/gcp/dataengine_lib_install.json | 2 +-
.../mock_response/gcp/dataengine_lib_list.json | 4 +-
.../mock_response/gcp/dataengine_start.json | 2 +-
.../mock_response/gcp/dataengine_stop.json | 2 +-
.../mock_response/gcp/dataengine_terminate.json | 2 +-
.../resources/mock_response/gcp/edge_create.json | 6 +-
.../resources/mock_response/gcp/edge_start.json | 2 +-
.../resources/mock_response/gcp/edge_stop.json | 2 +-
.../mock_response/gcp/edge_terminate.json | 2 +-
.../mock_response/gcp/notebook_create.json | 2 +-
.../mock_response/gcp/notebook_create_image.json | 2 +-
.../mock_response/gcp/notebook_git_creds.json | 2 +-
.../mock_response/gcp/notebook_lib_install.json | 6 +-
.../mock_response/gcp/notebook_lib_list.json | 4 +-
.../mock_response/gcp/notebook_start.json | 2 +-
.../mock_response/gcp/notebook_status.json | 14 +-
.../resources/mock_response/gcp/notebook_stop.json | 2 +-
.../mock_response/gcp/notebook_terminate.json | 2 +-
.../mock_response/gcp/project_create.json | 8 +-
.../mock_response/gcp/project_terminate.json | 2 +-
.../backendapi/core/DockerWarmuperTest.java | 133 ++
.../core/commands/CommandExecutorMockTest.java | 398 ++++++
.../docker/command/ImagesDockerCommandTest.java | 44 +
.../core/docker/command/RunDockerCommandTest.java | 319 +++++
.../folderlistener/FolderListenerTest.java | 190 +++
.../response/folderlistener/WatchItemListTest.java | 133 ++
.../response/folderlistener/WatchItemTest.java | 145 +++
.../dao/FileSystemCallbackHandlerDaoTest.java | 213 +++
.../RestoreCallbackHandlerServiceImplTest.java | 65 +
.../epam/datalab/rest/client/RESTServiceMock.java | 41 +
.../dlab/backendapi/core/DockerWarmuperTest.java | 129 --
.../core/commands/CommandExecutorMockTest.java | 398 ------
.../docker/command/ImagesDockerCommandTest.java | 44 -
.../core/docker/command/RunDockerCommandTest.java | 319 -----
.../folderlistener/FolderListenerTest.java | 190 ---
.../response/folderlistener/WatchItemListTest.java | 133 --
.../response/folderlistener/WatchItemTest.java | 145 ---
.../dao/FileSystemCallbackHandlerDaoTest.java | 208 ---
.../RestoreCallbackHandlerServiceImplTest.java | 63 -
.../com/epam/dlab/rest/client/RESTServiceMock.java | 41 -
services/readme.txt | 12 +-
services/self-service/entrypoint.sh | 2 +-
services/self-service/pom.xml | 56 +-
services/self-service/self-service.yml | 6 +-
.../datalab/backendapi/SelfServiceApplication.java | 217 ++++
.../epam/datalab/backendapi/annotation/Audit.java | 36 +
.../backendapi/annotation/BudgetLimited.java | 34 +
.../epam/datalab/backendapi/annotation/Info.java | 30 +
.../datalab/backendapi/annotation/Project.java | 31 +
.../backendapi/annotation/ProjectAdmin.java | 30 +
.../backendapi/annotation/ResourceName.java | 30 +
.../epam/datalab/backendapi/annotation/User.java | 30 +
.../backendapi/auth/KeycloakAuthenticator.java | 52 +
.../auth/SelfServiceSecurityAuthorizer.java | 34 +
.../filters/DropwizardBearerTokenFilterImpl.java | 39 +
.../backendapi/conf/CloudConfiguration.java | 59 +
.../backendapi/conf/KeycloakConfiguration.java | 27 +
.../conf/SelfServiceApplicationConfiguration.java | 264 ++++
.../com/epam/datalab/backendapi/dao/AuditDAO.java | 31 +
.../epam/datalab/backendapi/dao/AuditDAOImpl.java | 172 +++
.../com/epam/datalab/backendapi/dao/BackupDAO.java | 35 +
.../epam/datalab/backendapi/dao/BackupDAOImpl.java | 65 +
.../datalab/backendapi/dao/BaseBillingDAO.java | 252 ++++
.../com/epam/datalab/backendapi/dao/BaseDAO.java | 526 ++++++++
.../epam/datalab/backendapi/dao/BillingDAO.java | 53 +
.../datalab/backendapi/dao/ComputationalDAO.java | 403 ++++++
.../com/epam/datalab/backendapi/dao/DockerDAO.java | 43 +
.../epam/datalab/backendapi/dao/EndpointDAO.java | 53 +
.../datalab/backendapi/dao/EndpointDAOImpl.java | 90 ++
.../com/epam/datalab/backendapi/dao/EnvDAO.java | 512 ++++++++
.../datalab/backendapi/dao/ExploratoryDAO.java | 479 +++++++
.../datalab/backendapi/dao/ExploratoryLibDAO.java | 402 ++++++
.../epam/datalab/backendapi/dao/GitCredsDAO.java | 99 ++
.../backendapi/dao/ImageExploratoryDAO.java | 46 +
.../backendapi/dao/ImageExploratoryDAOImpl.java | 155 +++
.../epam/datalab/backendapi/dao/IndexCreator.java | 45 +
.../datalab/backendapi/dao/MongoCollections.java | 82 ++
.../epam/datalab/backendapi/dao/MongoSetting.java | 107 ++
.../epam/datalab/backendapi/dao/ProjectDAO.java | 59 +
.../datalab/backendapi/dao/ProjectDAOImpl.java | 190 +++
.../epam/datalab/backendapi/dao/RequestIdDAO.java | 68 +
.../datalab/backendapi/dao/SchedulerJobDAO.java | 248 ++++
.../epam/datalab/backendapi/dao/SecurityDAO.java | 118 ++
.../epam/datalab/backendapi/dao/SettingsDAO.java | 424 ++++++
.../epam/datalab/backendapi/dao/UserGroupDAO.java | 33 +
.../datalab/backendapi/dao/UserGroupDAOImpl.java | 68 +
.../epam/datalab/backendapi/dao/UserRoleDAO.java | 51 +
.../datalab/backendapi/dao/UserRoleDAOImpl.java | 215 ++++
.../datalab/backendapi/dao/UserSettingsDAO.java | 77 ++
.../datalab/backendapi/domain/AuditActionEnum.java | 24 +
.../datalab/backendapi/domain/AuditCreateDTO.java | 37 +
.../epam/datalab/backendapi/domain/AuditDTO.java | 39 +
.../backendapi/domain/AuditPaginationDTO.java | 43 +
.../backendapi/domain/AuditResourceTypeEnum.java | 24 +
.../backendapi/domain/AutoCompleteEnum.java | 24 +
.../datalab/backendapi/domain/BillingReport.java | 45 +
.../backendapi/domain/BillingReportLine.java | 56 +
.../epam/datalab/backendapi/domain/BudgetDTO.java | 36 +
.../backendapi/domain/CreateProjectDTO.java | 44 +
.../datalab/backendapi/domain/EndpointDTO.java | 50 +
.../backendapi/domain/EndpointResourcesDTO.java | 33 +
.../backendapi/domain/ExploratoryLibCache.java | 240 ++++
.../backendapi/domain/ExploratoryLibList.java | 250 ++++
.../domain/MavenSearchArtifactResponse.java | 82 ++
.../backendapi/domain/NotebookTemplate.java | 40 +
.../epam/datalab/backendapi/domain/ProjectDTO.java | 104 ++
.../backendapi/domain/ProjectEndpointDTO.java | 31 +
.../epam/datalab/backendapi/domain/RequestId.java | 95 ++
.../datalab/backendapi/domain/RequestIdDTO.java | 146 +++
.../domain/SchedulerConfigurationData.java | 31 +
.../backendapi/domain/UpdateProjectBudgetDTO.java | 35 +
.../backendapi/domain/UpdateProjectDTO.java | 40 +
.../dropwizard/bundles/DatalabKeycloakBundle.java | 58 +
.../dropwizard/listeners/MongoStartupListener.java | 97 ++
.../listeners/RestoreHandlerStartupListener.java | 50 +
.../backendapi/healthcheck/MongoHealthCheck.java | 46 +
.../backendapi/interceptor/AuditInterceptor.java | 127 ++
.../interceptor/BudgetLimitInterceptor.java | 64 +
.../interceptor/ProjectAdminInterceptor.java | 74 ++
.../backendapi/modules/CloudProviderModule.java | 112 ++
.../epam/datalab/backendapi/modules/DevModule.java | 197 +++
.../datalab/backendapi/modules/ModuleFactory.java | 51 +
.../backendapi/modules/ProductionModule.java | 176 +++
.../resources/ApplicationSettingResource.java | 73 ++
.../backendapi/resources/AuditResource.java | 70 +
.../backendapi/resources/BackupResource.java | 90 ++
.../backendapi/resources/BillingResource.java | 72 ++
.../backendapi/resources/BucketResource.java | 163 +++
.../backendapi/resources/EndpointResource.java | 151 +++
.../backendapi/resources/EnvironmentResource.java | 110 ++
.../backendapi/resources/ExploratoryResource.java | 175 +++
.../backendapi/resources/GitCredsResource.java | 78 ++
.../resources/ImageExploratoryResource.java | 110 ++
.../resources/InfrastructureInfoResource.java | 95 ++
.../resources/InfrastructureTemplateResource.java | 78 ++
.../backendapi/resources/KeycloakResource.java | 140 ++
.../resources/LibExploratoryResource.java | 217 ++++
.../backendapi/resources/ProjectResource.java | 244 ++++
.../backendapi/resources/SchedulerJobResource.java | 187 +++
.../backendapi/resources/SystemInfoResource.java | 56 +
.../backendapi/resources/UserGroupResource.java | 87 ++
.../backendapi/resources/UserRoleResource.java | 63 +
.../backendapi/resources/UserSettingsResource.java | 84 ++
.../resources/aws/ComputationalResourceAws.java | 272 ++++
.../resources/azure/AzureOauthResource.java | 60 +
.../azure/ComputationalResourceAzure.java | 181 +++
.../resources/callback/BackupCallback.java | 59 +
.../callback/CheckInactivityCallback.java | 69 +
.../resources/callback/ComputationalCallback.java | 96 ++
.../callback/EnvironmentStatusCallback.java | 71 +
.../resources/callback/ExploratoryCallback.java | 137 ++
.../resources/callback/GitCredsCallback.java | 64 +
.../resources/callback/ImageCallback.java | 71 +
.../resources/callback/LibraryCallback.java | 98 ++
.../resources/callback/ProjectCallback.java | 73 ++
.../resources/callback/ReuploadKeyCallback.java | 56 +
.../backendapi/resources/dto/BackupFormDTO.java | 40 +
.../backendapi/resources/dto/BackupInfoRecord.java | 45 +
.../backendapi/resources/dto/BillingFilter.java | 58 +
.../backendapi/resources/dto/BucketDeleteDTO.java | 38 +
.../resources/dto/BucketDownloadDTO.java | 35 +
.../resources/dto/ComputationalCreateFormDTO.java | 64 +
.../resources/dto/ComputationalTemplatesDTO.java | 35 +
.../resources/dto/ExploratoryActionFormDTO.java | 44 +
.../resources/dto/ExploratoryCreateFormDTO.java | 188 +++
.../resources/dto/ExploratoryCreatePopUp.java | 35 +
.../dto/ExploratoryImageCreateFormDTO.java | 39 +
.../backendapi/resources/dto/FolderUploadDTO.java | 35 +
.../datalab/backendapi/resources/dto/GroupDTO.java | 37 +
.../backendapi/resources/dto/HealthStatusDTO.java | 85 ++
.../backendapi/resources/dto/HealthStatusEnum.java | 44 +
.../resources/dto/HealthStatusPageDTO.java | 59 +
.../resources/dto/HealthStatusResource.java | 110 ++
.../backendapi/resources/dto/ImageInfoRecord.java | 37 +
.../datalab/backendapi/resources/dto/KeysDTO.java | 31 +
.../backendapi/resources/dto/LibInfoRecord.java | 43 +
.../resources/dto/LibInstallFormDTO.java | 48 +
.../datalab/backendapi/resources/dto/LibKey.java | 39 +
.../resources/dto/LibraryAutoCompleteDTO.java | 33 +
.../backendapi/resources/dto/LibraryDTO.java | 32 +
.../backendapi/resources/dto/LibraryStatus.java | 47 +
.../resources/dto/ProjectActionFormDTO.java | 36 +
.../resources/dto/ProjectInfrastructureInfo.java | 51 +
.../backendapi/resources/dto/QuotaUsageDTO.java | 32 +
.../resources/dto/SearchLibsFormDTO.java | 46 +
.../dto/SparkStandaloneClusterCreateForm.java | 42 +
.../dto/SparkStandaloneConfiguration.java | 33 +
.../backendapi/resources/dto/SystemInfoDto.java | 44 +
.../backendapi/resources/dto/UpdateGroupDTO.java | 38 +
.../resources/dto/UpdateRoleGroupDto.java | 36 +
.../resources/dto/UpdateUserGroupDto.java | 36 +
.../datalab/backendapi/resources/dto/UserDTO.java | 39 +
.../backendapi/resources/dto/UserGroupDto.java | 36 +
.../backendapi/resources/dto/UserResourceInfo.java | 57 +
.../backendapi/resources/dto/UserRoleDto.java | 64 +
.../dto/aws/AwsComputationalCreateForm.java | 57 +
.../resources/dto/aws/AwsEmrConfiguration.java | 49 +
.../dto/gcp/GcpComputationalCreateForm.java | 53 +
.../dto/gcp/GcpDataprocConfiguration.java | 42 +
.../resources/gcp/ComputationalResourceGcp.java | 259 ++++
.../backendapi/resources/gcp/GcpOauthResource.java | 65 +
.../epam/datalab/backendapi/roles/RoleType.java | 55 +
.../epam/datalab/backendapi/roles/UserRole.java | 142 ++
.../epam/datalab/backendapi/roles/UserRoles.java | 332 +++++
.../schedulers/CheckApplicationQuoteScheduler.java | 45 +
.../schedulers/CheckInactivityScheduledJob.java | 46 +
.../schedulers/CheckProjectQuoteScheduler.java | 52 +
.../schedulers/CheckUserQuoteScheduler.java | 49 +
.../schedulers/billing/BillingScheduler.java | 52 +
.../computational/StartComputationalJob.java | 44 +
.../computational/StopComputationalJob.java | 42 +
.../computational/TerminateComputationalJob.java | 41 +
.../endpoint/CheckEndpointStatusScheduler.java | 66 +
.../exploratory/StartExploratoryJob.java | 45 +
.../schedulers/exploratory/StopExploratoryJob.java | 46 +
.../schedulers/internal/ManagedScheduler.java | 115 ++
.../backendapi/schedulers/internal/Scheduled.java | 39 +
.../backendapi/service/AccessKeyService.java | 28 +
.../service/ApplicationSettingService.java | 31 +
.../service/ApplicationSettingServiceImpl.java | 46 +
.../datalab/backendapi/service/AuditService.java | 34 +
.../datalab/backendapi/service/BackupService.java | 37 +
.../datalab/backendapi/service/BillingService.java | 43 +
.../datalab/backendapi/service/BucketService.java | 39 +
.../backendapi/service/ComputationalService.java | 74 ++
.../backendapi/service/EndpointService.java | 48 +
.../backendapi/service/EnvironmentService.java | 46 +
.../backendapi/service/ExploratoryService.java | 62 +
.../backendapi/service/ExternalLibraryService.java | 28 +
.../backendapi/service/GitCredentialService.java | 29 +
.../backendapi/service/GuacamoleService.java | 30 +
.../service/ImageExploratoryService.java | 39 +
.../backendapi/service/InactivityService.java | 33 +
.../service/InfrastructureInfoService.java | 35 +
.../service/InfrastructureTemplateService.java | 32 +
.../backendapi/service/KeycloakService.java | 32 +
.../backendapi/service/KeycloakServiceImpl.java | 111 ++
.../datalab/backendapi/service/LibraryService.java | 42 +
.../datalab/backendapi/service/ProjectService.java | 61 +
.../backendapi/service/ReuploadKeyService.java | 28 +
.../backendapi/service/SchedulerJobService.java | 92 ++
.../backendapi/service/SecurityService.java | 30 +
.../backendapi/service/SecurityServiceImpl.java | 77 ++
.../backendapi/service/SystemInfoService.java | 27 +
.../datalab/backendapi/service/TagService.java | 29 +
.../datalab/backendapi/service/TagServiceImpl.java | 41 +
.../backendapi/service/UserGroupService.java | 37 +
.../backendapi/service/UserRoleService.java | 34 +
.../backendapi/service/UserRoleServiceImpl.java | 63 +
.../backendapi/service/UserSettingService.java | 35 +
.../backendapi/service/UserSettingServiceImpl.java | 48 +
.../service/impl/AccessKeyServiceImpl.java | 59 +
.../backendapi/service/impl/AuditServiceImpl.java | 63 +
.../backendapi/service/impl/BackupServiceImpl.java | 69 +
.../service/impl/BillingServiceImpl.java | 383 ++++++
.../backendapi/service/impl/BucketServiceImpl.java | 175 +++
.../service/impl/ComputationalServiceImpl.java | 396 ++++++
.../service/impl/EndpointServiceImpl.java | 193 +++
.../service/impl/EnvironmentServiceImpl.java | 233 ++++
.../service/impl/ExploratoryServiceImpl.java | 467 +++++++
.../service/impl/GitCredentialServiceImpl.java | 119 ++
.../service/impl/GuacamoleServiceImpl.java | 91 ++
.../service/impl/ImageExploratoryServiceImpl.java | 177 +++
.../service/impl/InactivityServiceImpl.java | 111 ++
.../impl/InfrastructureInfoServiceImpl.java | 193 +++
.../impl/InfrastructureTemplateServiceImpl.java | 246 ++++
.../service/impl/LibraryServiceImpl.java | 311 +++++
.../service/impl/MavenCentralLibraryService.java | 88 ++
.../service/impl/ProjectServiceImpl.java | 386 ++++++
.../service/impl/ReuploadKeyServiceImpl.java | 96 ++
.../service/impl/SchedulerJobServiceImpl.java | 514 ++++++++
.../service/impl/SystemInfoServiceImpl.java | 100 ++
.../service/impl/UserGroupServiceImpl.java | 228 ++++
.../servlet/guacamole/GuacamoleServlet.java | 97 ++
.../epam/datalab/backendapi/util/BillingUtils.java | 249 ++++
.../epam/datalab/backendapi/util/CSVFormatter.java | 67 +
.../datalab/backendapi/util/DateRemoverUtil.java | 40 +
.../epam/datalab/backendapi/util/KeycloakUtil.java | 41 +
.../datalab/backendapi/util/RequestBuilder.java | 653 ++++++++++
.../validation/MavenLibraryNameValidator.java | 41 +
.../validation/SchedulerJobDTOValidator.java | 47 +
.../validation/annotation/LibNameValid.java | 42 +
.../annotation/SchedulerJobDTOValid.java | 42 +
.../dlab/backendapi/SelfServiceApplication.java | 217 ----
.../com/epam/dlab/backendapi/annotation/Audit.java | 36 -
.../dlab/backendapi/annotation/BudgetLimited.java | 34 -
.../com/epam/dlab/backendapi/annotation/Info.java | 30 -
.../epam/dlab/backendapi/annotation/Project.java | 31 -
.../dlab/backendapi/annotation/ProjectAdmin.java | 30 -
.../dlab/backendapi/annotation/ResourceName.java | 30 -
.../com/epam/dlab/backendapi/annotation/User.java | 30 -
.../backendapi/auth/KeycloakAuthenticator.java | 52 -
.../auth/SelfServiceSecurityAuthorizer.java | 34 -
.../filters/DropwizardBearerTokenFilterImpl.java | 39 -
.../dlab/backendapi/conf/CloudConfiguration.java | 59 -
.../backendapi/conf/KeycloakConfiguration.java | 27 -
.../conf/SelfServiceApplicationConfiguration.java | 264 ----
.../com/epam/dlab/backendapi/dao/AuditDAO.java | 31 -
.../com/epam/dlab/backendapi/dao/AuditDAOImpl.java | 172 ---
.../com/epam/dlab/backendapi/dao/BackupDAO.java | 35 -
.../epam/dlab/backendapi/dao/BackupDAOImpl.java | 65 -
.../epam/dlab/backendapi/dao/BaseBillingDAO.java | 252 ----
.../java/com/epam/dlab/backendapi/dao/BaseDAO.java | 526 --------
.../com/epam/dlab/backendapi/dao/BillingDAO.java | 53 -
.../epam/dlab/backendapi/dao/ComputationalDAO.java | 403 ------
.../com/epam/dlab/backendapi/dao/DockerDAO.java | 40 -
.../com/epam/dlab/backendapi/dao/EndpointDAO.java | 53 -
.../epam/dlab/backendapi/dao/EndpointDAOImpl.java | 90 --
.../java/com/epam/dlab/backendapi/dao/EnvDAO.java | 512 --------
.../epam/dlab/backendapi/dao/ExploratoryDAO.java | 479 -------
.../dlab/backendapi/dao/ExploratoryLibDAO.java | 402 ------
.../com/epam/dlab/backendapi/dao/GitCredsDAO.java | 90 --
.../dlab/backendapi/dao/ImageExploratoryDAO.java | 46 -
.../backendapi/dao/ImageExploratoryDAOImpl.java | 155 ---
.../com/epam/dlab/backendapi/dao/IndexCreator.java | 42 -
.../epam/dlab/backendapi/dao/MongoCollections.java | 58 -
.../com/epam/dlab/backendapi/dao/MongoSetting.java | 107 --
.../com/epam/dlab/backendapi/dao/ProjectDAO.java | 59 -
.../epam/dlab/backendapi/dao/ProjectDAOImpl.java | 190 ---
.../com/epam/dlab/backendapi/dao/RequestIdDAO.java | 67 -
.../epam/dlab/backendapi/dao/SchedulerJobDAO.java | 248 ----
.../com/epam/dlab/backendapi/dao/SecurityDAO.java | 118 --
.../com/epam/dlab/backendapi/dao/SettingsDAO.java | 391 ------
.../com/epam/dlab/backendapi/dao/UserGroupDAO.java | 33 -
.../epam/dlab/backendapi/dao/UserGroupDAOImpl.java | 68 -
.../com/epam/dlab/backendapi/dao/UserRoleDAO.java | 51 -
.../epam/dlab/backendapi/dao/UserRoleDAOImpl.java | 215 ----
.../epam/dlab/backendapi/dao/UserSettingsDAO.java | 77 --
.../dlab/backendapi/domain/AuditActionEnum.java | 24 -
.../dlab/backendapi/domain/AuditCreateDTO.java | 37 -
.../com/epam/dlab/backendapi/domain/AuditDTO.java | 39 -
.../dlab/backendapi/domain/AuditPaginationDTO.java | 43 -
.../backendapi/domain/AuditResourceTypeEnum.java | 24 -
.../dlab/backendapi/domain/AutoCompleteEnum.java | 24 -
.../epam/dlab/backendapi/domain/BillingReport.java | 45 -
.../dlab/backendapi/domain/BillingReportLine.java | 56 -
.../com/epam/dlab/backendapi/domain/BudgetDTO.java | 36 -
.../dlab/backendapi/domain/CreateProjectDTO.java | 44 -
.../epam/dlab/backendapi/domain/EndpointDTO.java | 50 -
.../backendapi/domain/EndpointResourcesDTO.java | 33 -
.../backendapi/domain/ExploratoryLibCache.java | 240 ----
.../dlab/backendapi/domain/ExploratoryLibList.java | 250 ----
.../domain/MavenSearchArtifactResponse.java | 82 --
.../dlab/backendapi/domain/NotebookTemplate.java | 40 -
.../epam/dlab/backendapi/domain/ProjectDTO.java | 104 --
.../dlab/backendapi/domain/ProjectEndpointDTO.java | 31 -
.../com/epam/dlab/backendapi/domain/RequestId.java | 84 --
.../epam/dlab/backendapi/domain/RequestIdDTO.java | 121 --
.../domain/SchedulerConfigurationData.java | 31 -
.../backendapi/domain/UpdateProjectBudgetDTO.java | 35 -
.../dlab/backendapi/domain/UpdateProjectDTO.java | 40 -
.../dropwizard/bundles/DlabKeycloakBundle.java | 58 -
.../dropwizard/listeners/MongoStartupListener.java | 97 --
.../listeners/RestoreHandlerStartupListener.java | 50 -
.../backendapi/healthcheck/MongoHealthCheck.java | 46 -
.../backendapi/interceptor/AuditInterceptor.java | 127 --
.../interceptor/BudgetLimitInterceptor.java | 64 -
.../interceptor/ProjectAdminInterceptor.java | 74 --
.../backendapi/modules/CloudProviderModule.java | 112 --
.../epam/dlab/backendapi/modules/DevModule.java | 197 ---
.../dlab/backendapi/modules/ModuleFactory.java | 51 -
.../dlab/backendapi/modules/ProductionModule.java | 176 ---
.../resources/ApplicationSettingResource.java | 68 -
.../dlab/backendapi/resources/AuditResource.java | 70 -
.../dlab/backendapi/resources/BackupResource.java | 85 --
.../dlab/backendapi/resources/BillingResource.java | 72 --
.../dlab/backendapi/resources/BucketResource.java | 163 ---
.../backendapi/resources/EndpointResource.java | 151 ---
.../backendapi/resources/EnvironmentResource.java | 110 --
.../backendapi/resources/ExploratoryResource.java | 174 ---
.../backendapi/resources/GitCredsResource.java | 78 --
.../resources/ImageExploratoryResource.java | 110 --
.../resources/InfrastructureInfoResource.java | 95 --
.../resources/InfrastructureTemplateResource.java | 74 --
.../backendapi/resources/KeycloakResource.java | 140 --
.../resources/LibExploratoryResource.java | 217 ----
.../dlab/backendapi/resources/ProjectResource.java | 244 ----
.../backendapi/resources/SchedulerJobResource.java | 187 ---
.../backendapi/resources/SystemInfoResource.java | 56 -
.../backendapi/resources/UserGroupResource.java | 87 --
.../backendapi/resources/UserRoleResource.java | 63 -
.../backendapi/resources/UserSettingsResource.java | 79 --
.../resources/aws/ComputationalResourceAws.java | 272 ----
.../resources/azure/AzureOauthResource.java | 56 -
.../azure/ComputationalResourceAzure.java | 180 ---
.../resources/callback/BackupCallback.java | 59 -
.../callback/CheckInactivityCallback.java | 69 -
.../resources/callback/ComputationalCallback.java | 96 --
.../callback/EnvironmentStatusCallback.java | 71 -
.../resources/callback/ExploratoryCallback.java | 137 --
.../resources/callback/GitCredsCallback.java | 64 -
.../resources/callback/ImageCallback.java | 71 -
.../resources/callback/LibraryCallback.java | 98 --
.../resources/callback/ProjectCallback.java | 73 --
.../resources/callback/ReuploadKeyCallback.java | 56 -
.../backendapi/resources/dto/BackupFormDTO.java | 40 -
.../backendapi/resources/dto/BackupInfoRecord.java | 45 -
.../backendapi/resources/dto/BillingFilter.java | 58 -
.../backendapi/resources/dto/BucketDeleteDTO.java | 38 -
.../resources/dto/BucketDownloadDTO.java | 35 -
.../resources/dto/ComputationalCreateFormDTO.java | 64 -
.../resources/dto/ComputationalTemplatesDTO.java | 35 -
.../resources/dto/ExploratoryActionFormDTO.java | 44 -
.../resources/dto/ExploratoryCreateFormDTO.java | 188 ---
.../resources/dto/ExploratoryCreatePopUp.java | 35 -
.../dto/ExploratoryImageCreateFormDTO.java | 39 -
.../backendapi/resources/dto/FolderUploadDTO.java | 35 -
.../dlab/backendapi/resources/dto/GroupDTO.java | 37 -
.../backendapi/resources/dto/HealthStatusDTO.java | 72 --
.../backendapi/resources/dto/HealthStatusEnum.java | 42 -
.../resources/dto/HealthStatusPageDTO.java | 59 -
.../resources/dto/HealthStatusResource.java | 91 --
.../backendapi/resources/dto/ImageInfoRecord.java | 37 -
.../dlab/backendapi/resources/dto/KeysDTO.java | 31 -
.../backendapi/resources/dto/LibInfoRecord.java | 43 -
.../resources/dto/LibInstallFormDTO.java | 48 -
.../epam/dlab/backendapi/resources/dto/LibKey.java | 39 -
.../resources/dto/LibraryAutoCompleteDTO.java | 33 -
.../dlab/backendapi/resources/dto/LibraryDTO.java | 32 -
.../backendapi/resources/dto/LibraryStatus.java | 47 -
.../resources/dto/ProjectActionFormDTO.java | 36 -
.../resources/dto/ProjectInfrastructureInfo.java | 51 -
.../backendapi/resources/dto/QuotaUsageDTO.java | 32 -
.../resources/dto/SearchLibsFormDTO.java | 46 -
.../dto/SparkStandaloneClusterCreateForm.java | 42 -
.../dto/SparkStandaloneConfiguration.java | 33 -
.../backendapi/resources/dto/SystemInfoDto.java | 44 -
.../backendapi/resources/dto/UpdateGroupDTO.java | 38 -
.../resources/dto/UpdateRoleGroupDto.java | 36 -
.../resources/dto/UpdateUserGroupDto.java | 36 -
.../dlab/backendapi/resources/dto/UserDTO.java | 39 -
.../backendapi/resources/dto/UserGroupDto.java | 36 -
.../backendapi/resources/dto/UserResourceInfo.java | 57 -
.../dlab/backendapi/resources/dto/UserRoleDto.java | 64 -
.../dto/aws/AwsComputationalCreateForm.java | 57 -
.../resources/dto/aws/AwsEmrConfiguration.java | 49 -
.../dto/gcp/GcpComputationalCreateForm.java | 53 -
.../dto/gcp/GcpDataprocConfiguration.java | 42 -
.../resources/gcp/ComputationalResourceGcp.java | 259 ----
.../backendapi/resources/gcp/GcpOauthResource.java | 61 -
.../com/epam/dlab/backendapi/roles/RoleType.java | 55 -
.../com/epam/dlab/backendapi/roles/UserRole.java | 142 --
.../com/epam/dlab/backendapi/roles/UserRoles.java | 332 -----
.../schedulers/CheckApplicationQuoteScheduler.java | 45 -
.../schedulers/CheckInactivityScheduledJob.java | 46 -
.../schedulers/CheckProjectQuoteScheduler.java | 52 -
.../schedulers/CheckUserQuoteScheduler.java | 49 -
.../schedulers/billing/BillingScheduler.java | 52 -
.../computational/StartComputationalJob.java | 44 -
.../computational/StopComputationalJob.java | 42 -
.../computational/TerminateComputationalJob.java | 41 -
.../endpoint/CheckEndpointStatusScheduler.java | 66 -
.../exploratory/StartExploratoryJob.java | 45 -
.../schedulers/exploratory/StopExploratoryJob.java | 46 -
.../schedulers/internal/ManagedScheduler.java | 109 --
.../backendapi/schedulers/internal/Scheduled.java | 39 -
.../dlab/backendapi/service/AccessKeyService.java | 28 -
.../service/ApplicationSettingService.java | 31 -
.../service/ApplicationSettingServiceImpl.java | 45 -
.../epam/dlab/backendapi/service/AuditService.java | 34 -
.../dlab/backendapi/service/BackupService.java | 37 -
.../dlab/backendapi/service/BillingService.java | 43 -
.../dlab/backendapi/service/BucketService.java | 39 -
.../backendapi/service/ComputationalService.java | 74 --
.../dlab/backendapi/service/EndpointService.java | 48 -
.../backendapi/service/EnvironmentService.java | 46 -
.../backendapi/service/ExploratoryService.java | 62 -
.../backendapi/service/ExternalLibraryService.java | 28 -
.../backendapi/service/GitCredentialService.java | 29 -
.../dlab/backendapi/service/GuacamoleService.java | 30 -
.../service/ImageExploratoryService.java | 39 -
.../dlab/backendapi/service/InactivityService.java | 33 -
.../service/InfrastructureInfoService.java | 35 -
.../service/InfrastructureTemplateService.java | 32 -
.../dlab/backendapi/service/KeycloakService.java | 32 -
.../backendapi/service/KeycloakServiceImpl.java | 111 --
.../dlab/backendapi/service/LibraryService.java | 42 -
.../dlab/backendapi/service/ProjectService.java | 61 -
.../backendapi/service/ReuploadKeyService.java | 28 -
.../backendapi/service/SchedulerJobService.java | 92 --
.../dlab/backendapi/service/SecurityService.java | 30 -
.../backendapi/service/SecurityServiceImpl.java | 77 --
.../dlab/backendapi/service/SystemInfoService.java | 27 -
.../epam/dlab/backendapi/service/TagService.java | 29 -
.../dlab/backendapi/service/TagServiceImpl.java | 41 -
.../dlab/backendapi/service/UserGroupService.java | 37 -
.../dlab/backendapi/service/UserRoleService.java | 34 -
.../backendapi/service/UserRoleServiceImpl.java | 63 -
.../backendapi/service/UserSettingService.java | 35 -
.../backendapi/service/UserSettingServiceImpl.java | 48 -
.../service/impl/AccessKeyServiceImpl.java | 59 -
.../backendapi/service/impl/AuditServiceImpl.java | 63 -
.../backendapi/service/impl/BackupServiceImpl.java | 69 -
.../service/impl/BillingServiceImpl.java | 383 ------
.../backendapi/service/impl/BucketServiceImpl.java | 175 ---
.../service/impl/ComputationalServiceImpl.java | 396 ------
.../service/impl/EndpointServiceImpl.java | 193 ---
.../service/impl/EnvironmentServiceImpl.java | 233 ----
.../service/impl/ExploratoryServiceImpl.java | 467 -------
.../service/impl/GitCredentialServiceImpl.java | 119 --
.../service/impl/GuacamoleServiceImpl.java | 91 --
.../service/impl/ImageExploratoryServiceImpl.java | 177 ---
.../service/impl/InactivityServiceImpl.java | 111 --
.../impl/InfrastructureInfoServiceImpl.java | 193 ---
.../impl/InfrastructureTemplateServiceImpl.java | 246 ----
.../service/impl/LibraryServiceImpl.java | 311 -----
.../service/impl/MavenCentralLibraryService.java | 88 --
.../service/impl/ProjectServiceImpl.java | 386 ------
.../service/impl/ReuploadKeyServiceImpl.java | 96 --
.../service/impl/SchedulerJobServiceImpl.java | 514 --------
.../service/impl/SystemInfoServiceImpl.java | 100 --
.../service/impl/UserGroupServiceImpl.java | 228 ----
.../servlet/guacamole/GuacamoleServlet.java | 97 --
.../epam/dlab/backendapi/util/BillingUtils.java | 250 ----
.../epam/dlab/backendapi/util/CSVFormatter.java | 67 -
.../epam/dlab/backendapi/util/DateRemoverUtil.java | 40 -
.../epam/dlab/backendapi/util/KeycloakUtil.java | 41 -
.../epam/dlab/backendapi/util/RequestBuilder.java | 653 ----------
.../validation/MavenLibraryNameValidator.java | 41 -
.../validation/SchedulerJobDTOValidator.java | 47 -
.../validation/annotation/LibNameValid.java | 42 -
.../annotation/SchedulerJobDTOValid.java | 42 -
.../src/main/resources/mongo/aws/mongo_roles.json | 18 +-
.../main/resources/mongo/azure/mongo_roles.json | 12 +-
.../src/main/resources/mongo/gcp/mongo_roles.json | 20 +-
.../src/main/resources/quartz.properties | 2 +-
.../src/main/resources/webapp/angular.json | 4 +-
.../backup-dilog/backup-dilog.component.ts | 2 +-
.../manage-environment-dilog.component.html | 2 +-
.../manage-environment-dilog.component.ts | 2 +-
.../management-grid/management-grid.component.html | 4 +-
.../management-grid/management-grid.component.ts | 4 +-
.../management/management.component.html | 2 +-
.../ssn-monitor/ssn-monitor.component.ts | 2 +-
.../administration/project/project.component.ts | 2 +-
.../app/administration/roles/roles.component.ts | 2 +-
.../src/app/core/services/progress-bar.service.ts | 2 +-
.../accessnotebookguide.component.html | 15 +-
.../publickeyguide/publickeyguide.component.html | 2 +-
.../webapp/src/app/layout/layout.component.html | 2 +-
.../webapp/src/app/layout/layout.component.ts | 2 +-
.../webapp/src/app/login/login.component.css | 5 +-
.../webapp/src/app/login/login.component.html | 2 +-
.../webapp/src/app/login/login.component.ts | 2 +-
.../audit/audit-grid/audit-grid.component.ts | 2 +-
.../src/app/reports/audit/audit.component.ts | 14 +-
.../reporting-grid/reporting-grid.component.html | 12 +-
.../reporting-grid/reporting-grid.component.ts | 2 +-
.../app/reports/reporting/reporting.component.ts | 28 +-
.../reports/reporting/toolbar/toolbar.component.ts | 2 +-
.../bucket-browser/bucket-browser.component.html | 28 +-
.../bucket-browser/bucket-browser.component.ts | 2 +-
.../buckets-tree/bucket-tree.component.ts | 2 +-
.../folder-tree/folder-tree.component.ts | 2 +-
.../cluster-details/cluster-details.component.html | 97 +-
.../cluster-details/cluster-details.component.ts | 4 +-
...utational-resource-create-dialog.component.html | 145 ++-
...mputational-resource-create-dialog.component.ts | 14 +-
.../computational-resource-create.model.ts | 4 +-
.../computational-resources-list.component.html | 14 +-
.../ami-create-dialog.component.ts | 2 +-
.../create-environment.component.html | 32 +-
.../detail-dialog/detail-dialog.component.html | 40 +-
.../install-libraries.component.html | 2 +-
.../manage-ungit/manage-ungit.component.ts | 2 +-
.../resources-grid/resources-grid.component.html | 44 +-
.../src/app/resources/resources.component.ts | 2 +-
.../resources/scheduler/scheduler.component.html | 169 +--
.../app/resources/scheduler/scheduler.component.ts | 4 +-
.../access-denied/access-denied.component.ts | 2 +-
.../confirmation-dialog.component.html | 15 +-
.../confirmation-dialog.component.ts | 2 +-
.../src/app/shared/navbar/navbar.component.html | 2 +-
.../src/app/shared/navbar/navbar.component.scss | 2 +-
.../src/app/shared/navbar/navbar.component.ts | 30 +-
.../shared/time-picker/time-picker.component.ts | 2 +-
.../webapp/src/app/swagger/swagger.component.ts | 2 +-
.../src/app/webterminal/webterminal.component.ts | 4 +-
.../src/main/resources/webapp/src/assets/data.json | 4 +-
.../resources/webapp/src/assets/endpoint-api.json | 4 +-
.../webapp/src/assets/styles/app-loading.scss | 151 +--
.../webapp/src/assets/svg/datalab-logo.svg | 12 +
.../main/resources/webapp/src/assets/svg/logo.svg | 8 +
.../webapp/src/dictionary/aws.dictionary.ts | 8 +-
.../webapp/src/dictionary/azure.dictionary.ts | 6 +-
.../webapp/src/dictionary/gcp.dictionary.ts | 10 +-
.../webapp/src/dictionary/global.dictionary.ts | 4 +-
.../src/main/resources/webapp/src/favicon.ico | Bin 4286 -> 1414 bytes
.../src/main/resources/webapp/src/index.html | 38 +-
.../com/epam/{dlab => datalab}/backendapi/.gitkeep | 0
.../backendapi/domain/ExploratoryLibListTest.java | 67 +
.../resources/ApplicationSettingResourceTest.java | 105 ++
.../backendapi/resources/AuditResourceTest.java | 88 ++
.../backendapi/resources/BackupResourceTest.java | 228 ++++
.../resources/EnvironmentResourceTest.java | 291 +++++
.../resources/ExploratoryResourceTest.java | 325 +++++
.../backendapi/resources/GitCredsResourceTest.java | 180 +++
.../resources/ImageExploratoryResourceTest.java | 282 ++++
.../resources/InfrastructureInfoResourceTest.java | 205 +++
.../InfrastructureTemplateResourceTest.java | 182 +++
.../backendapi/resources/KeycloakResourceTest.java | 70 +
.../resources/LibExploratoryResourceTest.java | 443 +++++++
.../backendapi/resources/ProjectResourceTest.java | 292 +++++
.../resources/SchedulerJobResourceTest.java | 341 +++++
.../resources/SystemInfoResourceTest.java | 101 ++
.../datalab/backendapi/resources/TestBase.java | 92 ++
.../resources/UserGroupResourceTest.java | 186 +++
.../backendapi/resources/UserRoleResourceTest.java | 105 ++
.../resources/UserSettingsResourceTest.java | 166 +++
.../datalab/backendapi/roles/UserRolesTest.java | 88 ++
.../CheckApplicationQuoteSchedulerTest.java | 69 +
.../schedulers/CheckUserQuoteSchedulerTest.java | 81 ++
.../service/ApplicationSettingServiceImplTest.java | 74 ++
.../service/KeycloakServiceImplTest.java | 108 ++
.../backendapi/service/ProjectServiceImplTest.java | 426 ++++++
.../service/SecurityServiceImplTest.java | 134 ++
.../backendapi/service/TagServiceImplTest.java | 70 +
.../service/UserRoleServiceImplTest.java | 91 ++
.../service/UserSettingServiceImplTest.java | 89 ++
.../service/impl/AccessKeyServiceImplTest.java | 66 +
.../service/impl/AuditServiceImplTest.java | 106 ++
.../service/impl/BackupServiceImplTest.java | 151 +++
.../service/impl/BillingServiceImplTest.java | 739 +++++++++++
.../service/impl/BucketServiceImplTest.java | 264 ++++
.../service/impl/ComputationalServiceImplTest.java | 784 +++++++++++
.../service/impl/EndpointServiceImplTest.java | 310 +++++
.../service/impl/EnvironmentServiceImplTest.java | 369 ++++++
.../service/impl/ExploratoryServiceImplTest.java | 549 ++++++++
.../service/impl/GitCredentialServiceImplTest.java | 182 +++
.../impl/ImageExploratoryServiceImplTest.java | 352 +++++
.../impl/InfrastructureInfoServiceImplTest.java | 387 ++++++
.../InfrastructureTemplateServiceBaseTest.java | 226 ++++
.../service/impl/LibraryServiceImplTest.java | 490 +++++++
.../impl/MavenCentralLibraryServiceTest.java | 92 ++
.../service/impl/ReuploadKeyServiceImplTest.java | 183 +++
.../service/impl/SchedulerJobServiceImplTest.java | 1125 ++++++++++++++++
.../service/impl/SystemInfoServiceImplTest.java | 99 ++
.../service/impl/UserGroupServiceImplTest.java | 182 +++
.../datalab/backendapi/util/CSVFormatterTest.java | 46 +
.../backendapi/util/DateRemoverUtilTest.java | 44 +
.../backendapi/domain/ExploratoryLibListTest.java | 67 -
.../resources/ApplicationSettingResourceTest.java | 102 --
.../backendapi/resources/AuditResourceTest.java | 88 --
.../backendapi/resources/BackupResourceTest.java | 223 ----
.../resources/EnvironmentResourceTest.java | 291 -----
.../resources/ExploratoryResourceTest.java | 325 -----
.../backendapi/resources/GitCredsResourceTest.java | 174 ---
.../resources/ImageExploratoryResourceTest.java | 281 ----
.../resources/InfrastructureInfoResourceTest.java | 205 ---
.../InfrastructureTemplateResourceTest.java | 177 ---
.../backendapi/resources/KeycloakResourceTest.java | 70 -
.../resources/LibExploratoryResourceTest.java | 443 -------
.../backendapi/resources/ProjectResourceTest.java | 292 -----
.../resources/SchedulerJobResourceTest.java | 341 -----
.../resources/SystemInfoResourceTest.java | 97 --
.../epam/dlab/backendapi/resources/TestBase.java | 92 --
.../resources/UserGroupResourceTest.java | 186 ---
.../backendapi/resources/UserRoleResourceTest.java | 105 --
.../resources/UserSettingsResourceTest.java | 159 ---
.../epam/dlab/backendapi/roles/UserRolesTest.java | 88 --
.../CheckApplicationQuoteSchedulerTest.java | 66 -
.../schedulers/CheckUserQuoteSchedulerTest.java | 77 --
.../service/ApplicationSettingServiceImplTest.java | 72 --
.../service/KeycloakServiceImplTest.java | 108 --
.../backendapi/service/ProjectServiceImplTest.java | 426 ------
.../service/SecurityServiceImplTest.java | 134 --
.../backendapi/service/TagServiceImplTest.java | 70 -
.../service/UserRoleServiceImplTest.java | 91 --
.../service/UserSettingServiceImplTest.java | 86 --
.../service/impl/AccessKeyServiceImplTest.java | 66 -
.../service/impl/AuditServiceImplTest.java | 106 --
.../service/impl/BackupServiceImplTest.java | 151 ---
.../service/impl/BillingServiceImplTest.java | 739 -----------
.../service/impl/BucketServiceImplTest.java | 264 ----
.../service/impl/ComputationalServiceImplTest.java | 784 -----------
.../service/impl/EndpointServiceImplTest.java | 310 -----
.../service/impl/EnvironmentServiceImplTest.java | 369 ------
.../service/impl/ExploratoryServiceImplTest.java | 549 --------
.../service/impl/GitCredentialServiceImplTest.java | 175 ---
.../impl/ImageExploratoryServiceImplTest.java | 352 -----
.../impl/InfrastructureInfoServiceImplTest.java | 387 ------
.../InfrastructureTemplateServiceBaseTest.java | 226 ----
.../service/impl/LibraryServiceImplTest.java | 490 -------
.../impl/MavenCentralLibraryServiceTest.java | 90 --
.../service/impl/ReuploadKeyServiceImplTest.java | 183 ---
.../service/impl/SchedulerJobServiceImplTest.java | 1125 ----------------
.../service/impl/SystemInfoServiceImplTest.java | 96 --
.../service/impl/UserGroupServiceImplTest.java | 182 ---
.../dlab/backendapi/util/CSVFormatterTest.java | 46 -
.../dlab/backendapi/util/DateRemoverUtilTest.java | 44 -
2243 files changed, 81973 insertions(+), 81181 deletions(-)
diff --cc infrastructure-provisioning/scripts/deploy_datalab.py
index 0000000,8ebad3e..096150c
mode 000000,100644..100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@@ -1,0 -1,220 +1,231 @@@
+ #!/usr/bin/python
+ # *****************************************************************************
+ #
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing,
+ # software distributed under the License is distributed on an
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ # KIND, either express or implied. See the License for the
+ # specific language governing permissions and limitations
+ # under the License.
+ #
+ # ******************************************************************************
+
+
-from fabric.api import *
+ import argparse
+ import os
++from fabric.api import *
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--conf_service_base_name', type=str, help='unique name for DataLab environment')
-parser.add_argument('--conf_network_type', type=str, default='', help='Define in which network DataLab will be deployed. '
- 'Possible options: public|private')
++parser.add_argument('--conf_network_type', type=str, default='',
++ help='Define in which network DataLab will be deployed. '
++ 'Possible options: public|private')
+ parser.add_argument('--conf_vpc_cidr', type=str, default='', help='CIDR of VPC')
+ parser.add_argument('--conf_vpc2_cidr', type=str, default='', help='CIDR of secondary VPC')
+ parser.add_argument('--conf_allowed_ip_cidr', type=str, default='', help='Comma-separated CIDR of IPs which will have '
+ 'access to SSN')
+ parser.add_argument('--conf_user_subnets_range', type=str, default='', help='Range of subnets which will be using for '
+ 'users environments. For example: '
+ '10.10.0.0/24 - 10.10.10.0/24')
+ parser.add_argument('--conf_private_subnet_prefix', type=str, default='24', help='Private subnet prefix')
+ parser.add_argument('--conf_additional_tags', type=str, default='', help='Additional tags in format '
+ '"Key1:Value1;Key2:Value2"')
+ parser.add_argument('--conf_image_enabled', type=str, default='', help='Enable or Disable creating image at first time')
+ parser.add_argument('--aws_user_predefined_s3_policies', type=str, default='', help='Predefined policies for users '
+ 'instances')
+ parser.add_argument('--aws_access_key', type=str, default='', help='AWS Access Key ID')
+ parser.add_argument('--aws_secret_access_key', type=str, default='', help='AWS Secret Access Key')
+ parser.add_argument('--aws_region', type=str, default='', help='AWS region')
+ parser.add_argument('--aws_zone', type=str, default='', help='AWS zone')
+ parser.add_argument('--azure_region', type=str, default='', help='Azure region')
+ parser.add_argument('--gcp_region', type=str, default='', help='GCP region')
+ parser.add_argument('--gcp_zone', type=str, default='', help='GCP zone')
+ parser.add_argument('--conf_os_family', type=str, default='',
+ help='Operating system type. Available options: debian, redhat')
+ parser.add_argument('--conf_cloud_provider', type=str, default='',
+ help='Where DataLab should be deployed. Available options: aws, azure, gcp')
+ parser.add_argument('--ssn_hosted_zone_name', type=str, default='', help='Name of hosted zone')
+ parser.add_argument('--ssn_hosted_zone_id', type=str, default='', help='ID of hosted zone')
+ parser.add_argument('--ssn_subdomain', type=str, default='', help='Subdomain name')
+ parser.add_argument('--ssn_assume_role_arn', type=str, default='', help='Role ARN for creating Route53 record in '
+ 'different AWS account')
+ parser.add_argument('--ssl_cert_path', type=str, default='', help='Full path to SSL certificate')
+ parser.add_argument('--ssl_key_path', type=str, default='', help='Full path to key for SSL certificate')
+ parser.add_argument('--aws_vpc_id', type=str, default='', help='AWS VPC ID')
+ parser.add_argument('--conf_duo_vpc_enable', type=str, default='false', help='Duo VPC scheme enable(true|false)')
+ parser.add_argument('--aws_vpc2_id', type=str, default='', help='Secondary AWS VPC ID')
+ parser.add_argument('--aws_peering_id', type=str, default='', help='Amazon peering connection id')
+ parser.add_argument('--azure_vpc_name', type=str, default='', help='Azure VPC Name')
+ parser.add_argument('--gcp_vpc_name', type=str, default='', help='GCP VPC Name')
+ parser.add_argument('--aws_subnet_id', type=str, default='', help='AWS Subnet ID')
+ parser.add_argument('--azure_subnet_name', type=str, default='', help='Azure Subnet Name')
+ parser.add_argument('--gcp_subnet_name', type=str, default='', help='GCP Subnet Name')
+ parser.add_argument('--aws_security_groups_ids', type=str, default='', help='One of more comma-separated Security '
+ 'groups IDs for SSN')
+ parser.add_argument('--azure_security_group_name', type=str, default='', help='One of more comma-separated Security '
+ 'groups names for SSN')
+ parser.add_argument('--gcp_firewall_name', type=str, default='', help='One of more comma-separated GCP Firewall rules '
+ 'for SSN')
+ parser.add_argument('--key_path', type=str, default='', help='Path to admin key (WITHOUT KEY NAME)')
+ parser.add_argument('--conf_key_name', type=str, default='', help='Admin key name (WITHOUT ".pem")')
+ parser.add_argument('--workspace_path', type=str, default='', help='Admin key name (WITHOUT ".pem")')
+ parser.add_argument('--conf_tag_resource_id', type=str, default='datalab', help='The name of user tag')
+ parser.add_argument('--conf_billing_tag', type=str, default='datalab', help='Billing tag')
+ parser.add_argument('--aws_ssn_instance_size', type=str, default='t2.large', help='The SSN instance shape')
+ parser.add_argument('--azure_ssn_instance_size', type=str, default='Standard_DS2_v2', help='The SSN instance shape')
+ parser.add_argument('--gcp_ssn_instance_size', type=str, default='n1-standard-2', help='The SSN instance shape')
+ parser.add_argument('--aws_account_id', type=str, default='', help='The ID of Amazon account')
+ parser.add_argument('--aws_billing_bucket', type=str, default='', help='The name of S3 bucket where billing reports '
+ 'will be placed.')
+ parser.add_argument('--aws_job_enabled', type=str, default='false', help='Billing format. Available options: '
+ 'true (aws), false(epam)')
+ parser.add_argument('--aws_report_path', type=str, default='', help='The path to billing reports directory in S3 '
+ 'bucket')
+ parser.add_argument('--azure_resource_group_name', type=str, default='', help='Name of Resource group in Azure')
+ parser.add_argument('--azure_auth_path', type=str, default='', help='Full path to Azure credentials JSON file')
+ parser.add_argument('--azure_datalake_enable', type=str, default='', help='Provision DataLake storage account')
+ parser.add_argument('--azure_ad_group_id', type=str, default='', help='ID of Azure AD group')
+ parser.add_argument('--azure_offer_number', type=str, default='', help='Azure offer number')
+ parser.add_argument('--azure_currency', type=str, default='', help='Azure currency code')
+ parser.add_argument('--azure_locale', type=str, default='', help='Azure locale')
+ parser.add_argument('--azure_application_id', type=str, default='', help='Azure login application ID')
+ parser.add_argument('--azure_validate_permission_scope', type=str, default='true', help='Azure permission scope '
+ 'validation(true|false).')
+ parser.add_argument('--azure_oauth2_enabled', type=str, default='false', help='Using OAuth2 for logging in DataLab')
+ parser.add_argument('--azure_region_info', type=str, default='', help='Azure region info')
+ parser.add_argument('--azure_source_vpc_name', type=str, default='', help='Azure VPC source Name')
+ parser.add_argument('--azure_source_resource_group_name', type=str, default='', help='Azure source resource group')
+ parser.add_argument('--gcp_project_id', type=str, default='', help='The project ID in Google Cloud Platform')
+ parser.add_argument('--gcp_service_account_path', type=str, default='', help='The project ID in Google Cloud Platform')
-parser.add_argument('--datalab_id', type=str, default="'resource_tags_user_user_tag'", help='Column name in report file that contains '
- 'datalab id tag')
-parser.add_argument('--usage_date', type=str, default='line_item_usage_start_date', help='Column name in report file that contains '
- 'usage date tag')
-parser.add_argument('--product', type=str, default='product_product_name', help='Column name in report file that contains '
- 'product name tag')
-parser.add_argument('--usage_type', type=str, default='line_item_usage_type', help='Column name in report file that contains '
- 'usage type tag')
-parser.add_argument('--usage', type=str, default='line_item_usage_amount', help='Column name in report file that contains '
- 'usage tag')
-parser.add_argument('--cost', type=str, default='line_item_blended_cost', help='Column name in report file that contains cost tag')
-parser.add_argument('--resource_id', type=str, default='line_item_resource_id', help='Column name in report file that contains '
- 'datalab resource id tag')
++parser.add_argument('--datalab_id', type=str, default="'resource_tags_user_user_tag'",
++ help='Column name in report file that contains '
++ 'datalab id tag')
++parser.add_argument('--usage_date', type=str, default='line_item_usage_start_date',
++ help='Column name in report file that contains '
++ 'usage date tag')
++parser.add_argument('--product', type=str, default='product_product_name',
++ help='Column name in report file that contains '
++ 'product name tag')
++parser.add_argument('--usage_type', type=str, default='line_item_usage_type',
++ help='Column name in report file that contains '
++ 'usage type tag')
++parser.add_argument('--usage', type=str, default='line_item_usage_amount',
++ help='Column name in report file that contains '
++ 'usage tag')
++parser.add_argument('--cost', type=str, default='line_item_blended_cost',
++ help='Column name in report file that contains cost tag')
++parser.add_argument('--resource_id', type=str, default='line_item_resource_id',
++ help='Column name in report file that contains '
++ 'datalab resource id tag')
+ parser.add_argument('--ldap_hostname', type=str, default='localhost', help='Ldap instance hostname')
+ parser.add_argument('--ldap_dn', type=str, default='dc=example,dc=com',
+ help='Ldap distinguished name')
+ parser.add_argument('--ldap_ou', type=str, default='ou=People', help='Ldap organisation unit')
+ parser.add_argument('--ldap_service_username', type=str, default='cn=service-user', help='Ldap service user name')
+ parser.add_argument('--ldap_service_password', type=str, default='service-user-password',
+ help='Ldap password for admin user')
+ parser.add_argument('--keycloak_realm_name', type=str, default='datalab', help='Keycloak Realm name')
+ parser.add_argument('--keycloak_auth_server_url', type=str, default='datalab', help='Keycloak auth server URL')
+ parser.add_argument('--keycloak_client_name', type=str, default='datalab', help='Keycloak client name')
+ parser.add_argument('--keycloak_client_secret', type=str, default='datalab', help='Keycloak client secret')
+ parser.add_argument('--keycloak_user', type=str, default='datalab', help='Keycloak user')
-parser.add_argument('--keycloak_user_password', type=str, default='keycloak-user-password', help='Keycloak user password')
-parser.add_argument('--tags', type=str, default='line_item_operation,line_item_line_item_description', help='Column name in report file that '
- 'contains tags')
++parser.add_argument('--keycloak_user_password', type=str, default='keycloak-user-password',
++ help='Keycloak user password')
++parser.add_argument('--tags', type=str, default='line_item_operation,line_item_line_item_description',
++ help='Column name in report file that '
++ 'contains tags')
+ parser.add_argument('--billing_dataset_name', type=str, default='', help='Name of GCP dataset (BigQuery service)'
+ ' for billing')
+ parser.add_argument('--default_endpoint_name', type=str, default='local', help='Name of localhost provisioning service,'
+ 'that created by default')
+ parser.add_argument('--conf_stepcerts_enabled', type=str, default='false', help='Enable or disable step certificates')
+ parser.add_argument('--conf_stepcerts_root_ca', type=str, default='', help='Step root CA')
+ parser.add_argument('--conf_stepcerts_kid', type=str, default='', help='Step KID')
+ parser.add_argument('--conf_stepcerts_kid_password', type=str, default='', help='Step KID password')
+ parser.add_argument('--conf_stepcerts_ca_url', type=str, default='', help='Step CA URL')
-parser.add_argument('--conf_letsencrypt_enabled', type=str, default='false', help='Enable or disable Let`s Encrypt certificates')
++parser.add_argument('--conf_letsencrypt_enabled', type=str, default='false',
++ help='Enable or disable Let`s Encrypt certificates')
+ parser.add_argument('--conf_letsencrypt_domain_name', type=str, default='', help='Domain names to apply. '
- 'For multiple domains enter a comma separated list of domains as a parameter'
- 'ssn.domain_name will be used for ssn_node, DNS A record have to exist during deployment')
++ 'For multiple domains enter a comma separated list of domains as a parameter'
++ 'ssn.domain_name will be used for ssn_node, DNS A record have to exist during deployment')
+ parser.add_argument('--conf_letsencrypt_email', type=str, default='', help='Email that will be entered during '
- 'certificate obtaining and can be user for urgent renewal and security notices. '
- 'Use comma to register multiple emails, e.g. u1@example.com,u2@example.com.')
++ 'certificate obtaining and can be user for urgent renewal and security notices. '
++ 'Use comma to register multiple emails, e.g. u1@example.com,u2@example.com.')
+ parser.add_argument('--action', required=True, type=str, default='', choices=['build', 'deploy', 'create', 'terminate'],
+ help='Available options: build, deploy, create, terminate')
+ args = parser.parse_args()
+
+
+ def generate_docker_command():
+ docker_command = ''
+ command = []
+ if args.action == 'terminate':
+ command.append('sudo docker run -i ')
+ else:
+ command.append('sudo docker run -i -v {0}{1}.pem:/root/keys/{1}.pem -v {2}/web_app:/root/web_app '.
+ format(args.key_path, args.conf_key_name, args.workspace_path))
+ if args.conf_cloud_provider == 'azure':
+ command.append('-v {}:/root/azure_auth.json '.format(args.azure_auth_path))
+ elif args.conf_cloud_provider == 'gcp':
+ command.append('-v {}:/root/service_account.json '.format(args.gcp_service_account_path))
+ if args.ssl_cert_path != '' and args.ssl_key_path != '':
+ command.append('-v {}:/root/certs/datalab.crt -v {}:/root/certs/datalab.key '.format(args.ssl_cert_path,
- args.ssl_key_path))
++ args.ssl_key_path))
+ attrs = vars(args)
+ skipped_parameters = ['action', 'key_path', 'workspace_path', 'gcp_service_account_path', 'ssl_cert_path',
+ 'ssl_key_path']
+ for i in attrs:
+ if attrs[i] and i not in skipped_parameters:
+ command.append("-e '{}={}' ".format(i, attrs[i]))
+ command.append('-e "conf_resource=ssn" ')
+ command.append('docker.datalab-ssn ')
+ command.append('--action {} '.format(args.action))
+ return docker_command.join(command)
+
+
+ def build_docker_images(args):
+ # Building base and ssn docker images
+ with lcd(args.workspace_path):
+ local('sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
+ 'infrastructure-provisioning/src/general/files/{1}/'
+ 'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider))
+ local('sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
+ 'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider))
+
+
+ def deploy_datalab(args):
+ # Creating SSN node
+ docker_command = generate_docker_command()
+ local(docker_command)
+
+
+ def terminate_datalab(args):
+ # Dropping datalab environment with selected infrastructure tag
+ docker_command = generate_docker_command()
+ local(docker_command)
+
+
+ if __name__ == "__main__":
+ if not args.workspace_path:
+ print("Workspace path isn't set, using current directory: {}".format(os.environ['PWD']))
+ args.workspace_path = os.environ['PWD']
+ if args.action == 'build':
+ build_docker_images(args)
+ elif args.action == 'deploy':
+ deploy_datalab(args)
+ elif args.action == 'create':
+ build_docker_images(args)
+ deploy_datalab(args)
+ elif args.action == 'terminate':
+ build_docker_images(args)
+ terminate_datalab(args)
diff --cc infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
index 0c761a5,bc5e331..c3e738f
--- a/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
+++ b/infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py
@@@ -20,17 -20,17 +20,17 @@@
#
# ******************************************************************************
--from fabric.api import *
--from fabric.contrib.files import exists
import argparse
import boto3
--import traceback
--import sys
import json
--import time
--import string
import random
++import string
++import sys
++import time
++import traceback
from ConfigParser import SafeConfigParser
++from fabric.api import *
++from fabric.contrib.files import exists
parser = argparse.ArgumentParser()
parser.add_argument('--service_base_name', required=True, type=str, default='',
diff --cc infrastructure-provisioning/scripts/post-deployment_configuration.py
index 051258f,6098d62..7a39190
--- a/infrastructure-provisioning/scripts/post-deployment_configuration.py
+++ b/infrastructure-provisioning/scripts/post-deployment_configuration.py
@@@ -21,11 -21,11 +21,11 @@@
#
# ******************************************************************************
--from fabric.api import *
import argparse
import requests
import uuid
from Crypto.PublicKey import RSA
++from fabric.api import *
if __name__ == "__main__":
@@@ -43,13 -43,13 +43,19 @@@
}
print("Getting cloud and instance parameters")
-- server_external_ip = requests.get('http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip', headers=headers).text
- dlab_sbn = requests.get('http://metadata/computeMetadata/v1/instance/name', headers=headers).text
- dlab_ssn_static_ip_name = dlab_sbn + '-ip'
- dlab_zone = requests.get('http://metadata/computeMetadata/v1/instance/zone', headers=headers).text.split('/')[-1]
- dlab_region = '-'.join(dlab_zone.split('-', 2)[:2])
- deployment_vpcId = local("sudo gcloud compute instances describe {0} --zone {1} --format 'value(networkInterfaces.network)' | sed 's|.*/||'".format(dlab_sbn, dlab_zone), capture=True)
- deployment_subnetId = local("sudo gcloud compute instances describe {0} --zone {1} --format 'value(networkInterfaces.subnetwork)' | sed 's|.*/||'".format(dlab_sbn, dlab_zone), capture=True)
++ server_external_ip = requests.get(
++ 'http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip',
++ headers=headers).text
+ datalab_sbn = requests.get('http://metadata/computeMetadata/v1/instance/name', headers=headers).text
+ datalab_ssn_static_ip_name = datalab_sbn + '-ip'
+ datalab_zone = requests.get('http://metadata/computeMetadata/v1/instance/zone', headers=headers).text.split('/')[-1]
+ datalab_region = '-'.join(datalab_zone.split('-', 2)[:2])
- deployment_vpcId = local("sudo gcloud compute instances describe {0} --zone {1} --format 'value(networkInterfaces.network)' | sed 's|.*/||'".format(datalab_sbn, datalab_zone), capture=True)
- deployment_subnetId = local("sudo gcloud compute instances describe {0} --zone {1} --format 'value(networkInterfaces.subnetwork)' | sed 's|.*/||'".format(datalab_sbn, datalab_zone), capture=True)
++ deployment_vpcId = local(
++ "sudo gcloud compute instances describe {0} --zone {1} --format 'value(networkInterfaces.network)' | sed 's|.*/||'".format(
++ datalab_sbn, datalab_zone), capture=True)
++ deployment_subnetId = local(
++ "sudo gcloud compute instances describe {0} --zone {1} --format 'value(networkInterfaces.subnetwork)' | sed 's|.*/||'".format(
++ datalab_sbn, datalab_zone), capture=True)
gcp_projectId = requests.get('http://metadata/computeMetadata/v1/project/project-id', headers=headers).text
keycloak_redirectUri = 'http://{}'.format(server_external_ip)
@@@ -63,10 -63,10 +69,12 @@@
print("Generationg MongoDB password")
mongo_pwd = uuid.uuid4().hex
try:
- local("sudo echo -e 'db.changeUserPassword(\"admin\", \"{}\")' | mongo dlabdb --port 27017 -u admin -p MONGO_PASSWORD".format(mongo_pwd))
- local('sudo sed -i "s|MONGO_PASSWORD|{}|g" /opt/dlab/conf/billing.yml'.format(mongo_pwd))
- local("sudo echo -e 'db.changeUserPassword(\"admin\", \"{}\")' | mongo datalabdb --port 27017 -u admin -p MONGO_PASSWORD".format(mongo_pwd))
++ local(
++ "sudo echo -e 'db.changeUserPassword(\"admin\", \"{}\")' | mongo datalabdb --port 27017 -u admin -p MONGO_PASSWORD".format(
++ mongo_pwd))
+ local('sudo sed -i "s|MONGO_PASSWORD|{}|g" /opt/datalab/conf/billing.yml'.format(mongo_pwd))
- local('sudo sed -i "s|MONGO_PASSWORD|{}|g" /opt/dlab/conf/ssn.yml'.format(mongo_pwd))
+ local('sudo sed -i "s|MONGO_PASSWORD|{}|g" /opt/datalab/conf/ssn.yml'.format(mongo_pwd))
except:
print('Mongo password was already changed')
@@@ -76,63 -76,63 +84,97 @@@
if static_address_exist:
print('Address is already static')
else:
- local("sudo gcloud compute addresses create {0} --addresses {1} --region {2}".format(dlab_ssn_static_ip_name,
+ local("sudo gcloud compute addresses create {0} --addresses {1} --region {2}".format(datalab_ssn_static_ip_name,
server_external_ip,
- dlab_region), capture=True)
- datalab_region), capture=True)
++ datalab_region),
++ capture=True)
print("Overwriting SSN parameters")
if deployment_subnetId == 'default':
- local('sudo sed -i "s|# user_subnets_range|user_subnets_range|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini')
-
- local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/conf/self-service.yml'.format(dlab_sbn))
- local('sudo sed -i "s|KEYCLOAK_REDIRECTURI|{}|g" /opt/dlab/conf/self-service.yml'.format(keycloak_redirectUri))
- local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/conf/self-service.yml'.format(args.keycloak_realm_name))
- local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/conf/self-service.yml'.format(
- local('sudo sed -i "s|# user_subnets_range|user_subnets_range|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini')
++ local(
++ 'sudo sed -i "s|# user_subnets_range|user_subnets_range|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini')
+
+ local('sudo sed -i "s|DATALAB_SBN|{}|g" /opt/datalab/conf/self-service.yml'.format(datalab_sbn))
+ local('sudo sed -i "s|KEYCLOAK_REDIRECTURI|{}|g" /opt/datalab/conf/self-service.yml'.format(keycloak_redirectUri))
- local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/datalab/conf/self-service.yml'.format(args.keycloak_realm_name))
++ local(
++ 'sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/datalab/conf/self-service.yml'.format(args.keycloak_realm_name))
+ local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/datalab/conf/self-service.yml'.format(
args.keycloak_auth_server_url))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/conf/self-service.yml'.format(args.keycloak_client_name))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/self-service.yml'.format(
- local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/datalab/conf/self-service.yml'.format(args.keycloak_client_name))
++ local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/datalab/conf/self-service.yml'.format(
++ args.keycloak_client_name))
+ local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/datalab/conf/self-service.yml'.format(
args.keycloak_client_secret))
- local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_realm_name))
- local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/conf/provisioning.yml'.format(
- local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/datalab/conf/provisioning.yml'.format(args.keycloak_realm_name))
++ local(
++ 'sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/datalab/conf/provisioning.yml'.format(args.keycloak_realm_name))
+ local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/datalab/conf/provisioning.yml'.format(
args.keycloak_auth_server_url))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_client_name))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/provisioning.yml'.format(
- local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/datalab/conf/provisioning.yml'.format(args.keycloak_client_name))
++ local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/datalab/conf/provisioning.yml'.format(
++ args.keycloak_client_name))
+ local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/datalab/conf/provisioning.yml'.format(
args.keycloak_client_secret))
- local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/conf/provisioning.yml'.format(dlab_sbn))
- local('sudo sed -i "s|SUBNET_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(deployment_subnetId))
- local('sudo sed -i "s|DLAB_REGION|{}|g" /opt/dlab/conf/provisioning.yml'.format(dlab_region))
- local('sudo sed -i "s|DLAB_ZONE|{}|g" /opt/dlab/conf/provisioning.yml'.format(dlab_zone))
- local('sudo sed -i "s|SSN_VPC_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(deployment_vpcId))
- local('sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(gcp_projectId))
- local('sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_user))
- local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/conf/provisioning.yml'.format(
+ local('sudo sed -i "s|DATALAB_SBN|{}|g" /opt/datalab/conf/provisioning.yml'.format(datalab_sbn))
+ local('sudo sed -i "s|SUBNET_ID|{}|g" /opt/datalab/conf/provisioning.yml'.format(deployment_subnetId))
+ local('sudo sed -i "s|DATALAB_REGION|{}|g" /opt/datalab/conf/provisioning.yml'.format(datalab_region))
+ local('sudo sed -i "s|DATALAB_ZONE|{}|g" /opt/datalab/conf/provisioning.yml'.format(datalab_zone))
+ local('sudo sed -i "s|SSN_VPC_ID|{}|g" /opt/datalab/conf/provisioning.yml'.format(deployment_vpcId))
+ local('sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/datalab/conf/provisioning.yml'.format(gcp_projectId))
+ local('sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/datalab/conf/provisioning.yml'.format(args.keycloak_user))
+ local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/datalab/conf/provisioning.yml'.format(
args.keycloak_admin_password))
- local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/conf/billing.yml'.format(dlab_sbn))
-
- local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_sbn))
- local('sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(gcp_projectId))
- local('sudo sed -i "s|DLAB_REGION|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_region))
- local('sudo sed -i "s|DLAB_ZONE|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_zone))
- local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_realm_name))
- local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_auth_server_url))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_client_name))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_client_secret))
- local('sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_user))
- local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_admin_password))
+ local('sudo sed -i "s|DATALAB_SBN|{}|g" /opt/datalab/conf/billing.yml'.format(datalab_sbn))
+
- local('sudo sed -i "s|DATALAB_SBN|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(datalab_sbn))
- local('sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(gcp_projectId))
- local('sudo sed -i "s|DATALAB_REGION|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(datalab_region))
- local('sudo sed -i "s|DATALAB_ZONE|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(datalab_zone))
- local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_realm_name))
- local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_auth_server_url))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_client_name))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_client_secret))
- local('sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_user))
- local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_admin_password))
++ local(
++ 'sudo sed -i "s|DATALAB_SBN|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ datalab_sbn))
++ local(
++ 'sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ gcp_projectId))
++ local(
++ 'sudo sed -i "s|DATALAB_REGION|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ datalab_region))
++ local(
++ 'sudo sed -i "s|DATALAB_ZONE|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ datalab_zone))
++ local(
++ 'sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ args.keycloak_realm_name))
++ local(
++ 'sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ args.keycloak_auth_server_url))
++ local(
++ 'sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ args.keycloak_client_name))
++ local(
++ 'sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ args.keycloak_client_secret))
++ local(
++ 'sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ args.keycloak_user))
++ local(
++ 'sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/datalab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
++ args.keycloak_admin_password))
print('SSL certificate generating')
keystore_passwd = uuid.uuid4().hex
- local('sudo rm /home/dlab-user/keys/ssn*')
- local('sudo rm /etc/ssl/certs/dlab*')
- local('sudo keytool -delete -noprompt -trustcacerts -alias ssn -storepass changeit -keystore /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/security/cacerts')
- local('sudo openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/dlab.key -out /etc/ssl/certs/dlab.crt -subj "/C=US/ST=US/L=US/O=dlab/CN=localhost/subjectAltName={0}"'.format(server_external_ip))
- local('sudo openssl pkcs12 -export -in /etc/ssl/certs/dlab.crt -inkey /etc/ssl/certs/dlab.key -name ssn -out /home/dlab-user/keys/ssn.p12 -password pass:{0}'.format(keystore_passwd))
- local('sudo keytool -importkeystore -srckeystore /home/dlab-user/keys/ssn.p12 -srcstoretype PKCS12 -alias ssn -destkeystore /home/dlab-user/keys/ssn.keystore.jks -deststorepass {0} -srcstorepass {0}'.format(keystore_passwd))
- local('sudo keytool -importcert -trustcacerts -alias ssn -file /etc/ssl/certs/dlab.crt -noprompt -storepass changeit -keystore /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/security/cacerts')
- local('sudo sed -i "s|KEYSTORE_PASSWORD|{}|g" /opt/dlab/conf/ssn.yml'.format(keystore_passwd))
+ local('sudo rm /home/datalab-user/keys/ssn*')
+ local('sudo rm /etc/ssl/certs/datalab*')
- local('sudo keytool -delete -noprompt -trustcacerts -alias ssn -storepass changeit -keystore /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/security/cacerts')
- local('sudo openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/datalab.key -out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN=localhost/subjectAltName={0}"'.format(server_external_ip))
- local('sudo openssl pkcs12 -export -in /etc/ssl/certs/datalab.crt -inkey /etc/ssl/certs/datalab.key -name ssn -out /home/datalab-user/keys/ssn.p12 -password pass:{0}'.format(keystore_passwd))
- local('sudo keytool -importkeystore -srckeystore /home/datalab-user/keys/ssn.p12 -srcstoretype PKCS12 -alias ssn -destkeystore /home/datalab-user/keys/ssn.keystore.jks -deststorepass {0} -srcstorepass {0}'.format(keystore_passwd))
- local('sudo keytool -importcert -trustcacerts -alias ssn -file /etc/ssl/certs/datalab.crt -noprompt -storepass changeit -keystore /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/security/cacerts')
++ local(
++ 'sudo keytool -delete -noprompt -trustcacerts -alias ssn -storepass changeit -keystore /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/security/cacerts')
++ local(
++ 'sudo openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/datalab.key -out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN=localhost/subjectAltName={0}"'.format(
++ server_external_ip))
++ local(
++ 'sudo openssl pkcs12 -export -in /etc/ssl/certs/datalab.crt -inkey /etc/ssl/certs/datalab.key -name ssn -out /home/datalab-user/keys/ssn.p12 -password pass:{0}'.format(
++ keystore_passwd))
++ local(
++ 'sudo keytool -importkeystore -srckeystore /home/datalab-user/keys/ssn.p12 -srcstoretype PKCS12 -alias ssn -destkeystore /home/datalab-user/keys/ssn.keystore.jks -deststorepass {0} -srcstorepass {0}'.format(
++ keystore_passwd))
++ local(
++ 'sudo keytool -importcert -trustcacerts -alias ssn -file /etc/ssl/certs/datalab.crt -noprompt -storepass changeit -keystore /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/security/cacerts')
+ local('sudo sed -i "s|KEYSTORE_PASSWORD|{}|g" /opt/datalab/conf/ssn.yml'.format(keystore_passwd))
print('Nginx configuration updating')
local('sudo sed -i "s|SERVER_IP|{}|g" /etc/nginx/conf.d/nginx_proxy.conf'.format(server_external_ip))
@@@ -140,8 -140,8 +182,8 @@@
local('sudo supervisorctl restart all')
print('Rebuilding docker images')
- local('cd /opt/dlab/sources/infrastructure-provisioning/src/ && sudo docker-build all')
+ local('cd /opt/datalab/sources/infrastructure-provisioning/src/ && sudo docker-build all')
print('[SUMMARY]')
- print('Mongo password stored in /opt/dlab/conf/ssn.yml')
- print('SSH key for dlab-user stored in /home/dlab-user/keys/KEY-FILE.pem')
+ print('Mongo password stored in /opt/datalab/conf/ssn.yml')
- print('SSH key for datalab-user stored in /home/datalab-user/keys/KEY-FILE.pem')
++ print('SSH key for datalab-user stored in /home/datalab-user/keys/KEY-FILE.pem')
diff --cc infrastructure-provisioning/src/base/scripts/install_prerequisites.py
index e0c3c85,432dd2a..5c95894
--- a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
+++ b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
@@@ -21,20 -21,20 +21,19 @@@
#
# ******************************************************************************
--from fabric.api import *
import argparse
import json
- from dlab.fab import *
- from dlab.common_lib import *
- from fabric.contrib.files import exists
- import sys
-from datalab.fab import *
+import os
-
+ from datalab.common_lib import *
++from datalab.fab import *
++from fabric.api import *
+ from fabric.contrib.files import exists
-import sys
-import os
-
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
parser.add_argument('--keyfile', type=str, default='')
--parser.add_argument('--pip_packages', type=str, default='boto3 argparse fabric==1.14.0 awscli google-api-python-client google-auth-httplib2 google-cloud-storage pycrypto azure==2.0.0')
++parser.add_argument('--pip_packages', type=str,
++ default='boto3 argparse fabric==1.14.0 awscli google-api-python-client google-auth-httplib2 google-cloud-storage pycrypto azure==2.0.0')
parser.add_argument('--additional_config', type=str, default='{"empty":"string"}')
parser.add_argument('--user', type=str, default='')
parser.add_argument('--edge_private_ip', type=str, default='')
diff --cc infrastructure-provisioning/src/base/scripts/install_user_key.py
index f079707,ae3bbdc..74e22de
--- a/infrastructure-provisioning/src/base/scripts/install_user_key.py
+++ b/infrastructure-provisioning/src/base/scripts/install_user_key.py
@@@ -21,13 -21,13 +21,11 @@@
#
# ******************************************************************************
--from fabric.api import *
--from fabric.contrib.files import exists
- from dlab.fab import *
-from datalab.fab import *
import argparse
import json
import sys
--import os
++from datalab.fab import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
diff --cc infrastructure-provisioning/src/dataengine-service/fabfile.py
index c21555b,ee1ce6b..8f475b4
--- a/infrastructure-provisioning/src/dataengine-service/fabfile.py
+++ b/infrastructure-provisioning/src/dataengine-service/fabfile.py
@@@ -21,21 -21,21 +21,20 @@@
#
# ******************************************************************************
--import json
--import time
--from fabric.api import *
- from dlab.fab import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
-from datalab.fab import *
-from datalab.meta_lib import *
-from datalab.actions_lib import *
--import sys
++import logging
import os
++import sys
import uuid
--import logging
++from datalab.actions_lib import *
++from datalab.fab import *
++from datalab.meta_lib import *
++from fabric.api import *
def run():
-- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
-- local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
++ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
++ os.environ['request_id'])
++ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.INFO,
filename=local_log_filepath)
diff --cc infrastructure-provisioning/src/dataengine/fabfile.py
index 319c6f1,6bd4947..f5775a0
--- a/infrastructure-provisioning/src/dataengine/fabfile.py
+++ b/infrastructure-provisioning/src/dataengine/fabfile.py
@@@ -21,21 -21,21 +21,19 @@@
#
# ******************************************************************************
--import json
--import time
--from fabric.api import *
- from dlab.fab import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
- import sys
- import os
- import uuid
+import logging
++import os
++import sys
++from datalab.actions_lib import *
+ from datalab.fab import *
+ from datalab.meta_lib import *
-from datalab.actions_lib import *
-import sys
-import os
-import uuid
-import logging
++from fabric.api import *
def run():
-- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
-- local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
++ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
++ os.environ['request_id'])
++ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.INFO,
filename=local_log_filepath)
diff --cc infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
index 2089613,0432d24..50d9483
--- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
+++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
@@@ -22,13 -22,13 +22,11 @@@
# ******************************************************************************
import argparse
--import json
- import sys
- from dlab.notebook_lib import *
- from dlab.actions_lib import *
- from dlab.fab import *
+import os
-
+ import sys
-from datalab.notebook_lib import *
+ from datalab.actions_lib import *
+ from datalab.fab import *
-import os
-
++from datalab.notebook_lib import *
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
diff --cc infrastructure-provisioning/src/deeplearning/fabfile.py
index cd9fbb5,c92efc2..ad090d2
--- a/infrastructure-provisioning/src/deeplearning/fabfile.py
+++ b/infrastructure-provisioning/src/deeplearning/fabfile.py
@@@ -22,18 -22,18 +22,18 @@@
# ******************************************************************************
import logging
--import json
- import sys
- from dlab.fab import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
+import os
+ import sys
+import uuid
++from datalab.actions_lib import *
+ from datalab.fab import *
+ from datalab.meta_lib import *
-from datalab.actions_lib import *
-import os
-import uuid
# Main function for provisioning notebook server
def run():
-- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
++ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
++ os.environ['request_id'])
local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
diff --cc infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index b2e93d0,27bd5f1..cfbfb6c
--- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@@ -21,15 -21,15 +21,14 @@@
#
# ******************************************************************************
- from dlab.actions_lib import *
- from dlab.common_lib import *
- from dlab.notebook_lib import *
- from dlab.fab import *
- from fabric.api import *
- from fabric.contrib.files import exists
+import argparse
+import os
-
+ from datalab.actions_lib import *
+ from datalab.common_lib import *
-from datalab.notebook_lib import *
+ from datalab.fab import *
++from datalab.notebook_lib import *
+ from fabric.api import *
+ from fabric.contrib.files import exists
-import argparse
-import os
-
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
diff --cc infrastructure-provisioning/src/edge/fabfile.py
index 66a656b,7260ad6..01ff3b0
--- a/infrastructure-provisioning/src/edge/fabfile.py
+++ b/infrastructure-provisioning/src/edge/fabfile.py
@@@ -21,17 -21,17 +21,17 @@@
#
# ******************************************************************************
--import json
--from fabric.api import *
import logging
--import sys
import os
- from dlab.fab import *
-from datalab.fab import *
++import sys
import traceback
++from datalab.fab import *
++from fabric.api import *
def status():
-- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
++ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
++ os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
diff --cc infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
index bad50ad,56861b8..0adb505
--- a/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
+++ b/infrastructure-provisioning/src/edge/scripts/configure_http_proxy.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--from fabric.api import *
--from fabric.contrib.files import exists
- from dlab.edge_lib import configure_http_proxy_server
-from datalab.edge_lib import configure_http_proxy_server
import argparse
import json
import sys
++from datalab.edge_lib import configure_http_proxy_server
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
diff --cc infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py
index b7123a3,4b4de0d..8b67a41
--- a/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py
+++ b/infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py
@@@ -21,12 -21,12 +21,12 @@@
#
# ******************************************************************************
--import logging
--from fabric.api import *
import argparse
--import sys
++import logging
import os
- from dlab.edge_lib import install_nginx_ldap
++import sys
+ from datalab.edge_lib import install_nginx_ldap
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
diff --cc infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py
index b853af5,a19eb67..bef9ec9
--- a/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py
+++ b/infrastructure-provisioning/src/edge/scripts/reupload_ssh_key.py
@@@ -21,15 -21,15 +21,14 @@@
#
# ******************************************************************************
++import logging
import os
import sys
--import logging
import traceback
- from dlab.fab import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
++from datalab.actions_lib import *
+ from datalab.fab import *
+ from datalab.meta_lib import *
-from datalab.actions_lib import *
from fabric.api import *
--import multiprocessing
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
diff --cc infrastructure-provisioning/src/general/api/check_inactivity.py
index 31d7bb3,bddd150..e07609b
--- a/infrastructure-provisioning/src/general/api/check_inactivity.py
+++ b/infrastructure-provisioning/src/general/api/check_inactivity.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,9 -49,9 +48,9 @@@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/configure.py
index 4d6cc5f,ada70b2..691f3db
--- a/infrastructure-provisioning/src/general/api/configure.py
+++ b/infrastructure-provisioning/src/general/api/configure.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -48,9 -48,9 +47,9 @@@
reply['response']['result'] = json.loads(f.read())
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/create.py
index b2437b0,352e06b..e0910dc
--- a/infrastructure-provisioning/src/general/api/create.py
+++ b/infrastructure-provisioning/src/general/api/create.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -55,9 -55,9 +54,9 @@@
with open("/response/{}.json".format(os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))
else:
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/create_image.py
index 425cf26,501adf4..df85de9
--- a/infrastructure-provisioning/src/general/api/create_image.py
+++ b/infrastructure-provisioning/src/general/api/create_image.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,9 -49,9 +48,9 @@@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/git_creds.py
index 4edd370,c785fa0..a5a7e1b
--- a/infrastructure-provisioning/src/general/api/git_creds.py
+++ b/infrastructure-provisioning/src/general/api/git_creds.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,9 -49,9 +48,9 @@@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/install_libs.py
index 21489a9,823dc4b..3f818a5
--- a/infrastructure-provisioning/src/general/api/install_libs.py
+++ b/infrastructure-provisioning/src/general/api/install_libs.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,9 -49,9 +48,9 @@@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/list_libs.py
index d7c97a9,9438fe0..5346b27
--- a/infrastructure-provisioning/src/general/api/list_libs.py
+++ b/infrastructure-provisioning/src/general/api/list_libs.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,13 -49,13 +48,14 @@@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
- reply['response']['result']['file'] = "/opt/dlab/tmp/result/{0}_{1}_{2}_all_pkgs.json".format(os.environ['project_name'],
- reply['response']['result']['file'] = "/opt/datalab/tmp/result/{0}_{1}_{2}_all_pkgs.json".format(os.environ['project_name'],
-- os.environ['application'],
-- os.environ['request_id'])
++ reply['response']['result']['file'] = "/opt/datalab/tmp/result/{0}_{1}_{2}_all_pkgs.json".format(
++ os.environ['project_name'],
++ os.environ['application'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/reconfigure_spark.py
index 09139ca,dd98f44..3e53885
--- a/infrastructure-provisioning/src/general/api/reconfigure_spark.py
+++ b/infrastructure-provisioning/src/general/api/reconfigure_spark.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,9 -49,9 +48,9 @@@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/recreate.py
index e91e579,a51535a..63742e1
--- a/infrastructure-provisioning/src/general/api/recreate.py
+++ b/infrastructure-provisioning/src/general/api/recreate.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -55,9 -55,9 +54,9 @@@
with open("/response/{}.json".format(os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))
else:
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/reupload_key.py
index d4af110,f8b4e28..7f9b29a
--- a/infrastructure-provisioning/src/general/api/reupload_key.py
+++ b/infrastructure-provisioning/src/general/api/reupload_key.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,11 -49,11 +48,9 @@@
result = {"error": "Failed to open result.json"}
reply['error_message'] = 'Failed to open result.json'
--
--
- log = "/var/log/dlab/edge/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ log = "/var/log/datalab/edge/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
try:
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/start.py
index 469d5a6,0522363..94e92b1
--- a/infrastructure-provisioning/src/general/api/start.py
+++ b/infrastructure-provisioning/src/general/api/start.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -48,9 -48,9 +47,9 @@@
reply['response']['result'] = json.loads(f.read())
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/status.py
index 0b007b9,70b7961..97a767d
--- a/infrastructure-provisioning/src/general/api/status.py
+++ b/infrastructure-provisioning/src/general/api/status.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -48,9 -48,9 +47,9 @@@
reply['response']['result'] = json.loads(f.read())
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['edge_user_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['edge_user_name'],
- os.environ['request_id'])
++ os.environ['edge_user_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['edge_user_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/stop.py
index 8dc2a11,26468af..7cc0123
--- a/infrastructure-provisioning/src/general/api/stop.py
+++ b/infrastructure-provisioning/src/general/api/stop.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,9 -49,9 +48,9 @@@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/terminate.py
index 933af27,dde8fc5..2ba94bb
--- a/infrastructure-provisioning/src/general/api/terminate.py
+++ b/infrastructure-provisioning/src/general/api/terminate.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -55,9 -55,9 +54,9 @@@
with open("/response/{}.json".format(os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))
else:
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/api/terminate_image.py
index 0700686,aea7934..d88d3bc
--- a/infrastructure-provisioning/src/general/api/terminate_image.py
+++ b/infrastructure-provisioning/src/general/api/terminate_image.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
import json
++import os
import sys
from fabric.api import local
--
if __name__ == "__main__":
success = True
try:
@@@ -49,9 -49,9 +48,9 @@@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ reply['response']['log'] = "/var/log/datalab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
++ os.environ['project_name'],
++ os.environ['request_id'])
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
diff --cc infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 638ba1d,c322342..ed89a4b
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@@ -19,26 -19,26 +19,25 @@@
#
# ******************************************************************************
++import ast
++import backoff
import boto3
import botocore
--from botocore.client import Config
--import backoff
--from botocore.exceptions import ClientError
--import time
--import sys
--import os
++import datalab.fab
import json
--from fabric.api import *
--from fabric.contrib.files import exists
import logging
- from dlab.meta_lib import *
- from dlab.fab import *
-from datalab.meta_lib import *
-from datalab.fab import *
++import meta_lib
++import os
++import sys
++import time
import traceback
import urllib2
--import meta_lib
- import dlab.fab
-import datalab.fab
import uuid
--import ast
++from botocore.client import Config
++from datalab.fab import *
++from datalab.meta_lib import *
++from fabric.api import *
++from fabric.contrib.files import exists
def backoff_log(err):
@@@ -1831,8 -1831,8 +1830,8 @@@ def configure_dataengine_spark(cluster_
local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf {1}spark/conf/spark-defaults.conf'.format(cluster_name,
cluster_dir))
if spark_configs and os.path.exists('{0}'.format(cluster_dir)):
- dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
- capture=True)
+ datalab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
- capture=True)
++ capture=True)
spark_configurations = ast.literal_eval(spark_configs)
new_spark_defaults = list()
spark_defaults = local('cat {0}spark/conf/spark-defaults.conf'.format(cluster_dir), capture=True)
diff --cc infrastructure-provisioning/src/general/lib/aws/meta_lib.py
index c852ffc,36d6bcc..e0a31c8
--- a/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/meta_lib.py
@@@ -19,18 -19,18 +19,16 @@@
#
# ******************************************************************************
++import actions_lib
++import backoff
import boto3
--from botocore.client import Config
--import json, urllib2
--import time
++import json
import logging
--import traceback
import sys
--import backoff
--import random
--import string
- from dlab.fab import *
- import actions_lib
++import time
++import traceback
++from botocore.client import Config
+ from datalab.fab import *
-import actions_lib
def get_instance_hostname(tag_name, instance_name):
diff --cc infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 2e97003,ed914fd..d11e223
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@@ -19,31 -19,31 +19,30 @@@
#
# ******************************************************************************
--from azure.common.client_factory import get_client_from_auth_file
++import ast
import azure.common
++import backoff
++import datalab.common_lib
++import datalab.fab
++import json
++import logging
++import meta_lib
++import os
++import sys
++import time
++import traceback
++import urllib2
++from azure.common.client_factory import get_client_from_auth_file
++from azure.datalake.store import core, lib
from azure.mgmt.authorization import AuthorizationManagementClient
from azure.mgmt.compute import ComputeManagementClient
--from azure.mgmt.resource import ResourceManagementClient
++from azure.mgmt.datalake.store import DataLakeStoreAccountManagementClient
from azure.mgmt.network import NetworkManagementClient
++from azure.mgmt.resource import ResourceManagementClient
from azure.mgmt.storage import StorageManagementClient
from azure.storage.blob import BlockBlobService
--from azure.mgmt.datalake.store import DataLakeStoreAccountManagementClient
--from azure.datalake.store import core, lib
--from azure.graphrbac import GraphRbacManagementClient
--from azure.common.credentials import ServicePrincipalCredentials
--import azure.common.exceptions as AzureExceptions
from fabric.api import *
from fabric.contrib.files import exists
--import backoff
--import urllib2
--import meta_lib
--import logging
--import traceback
--import sys, time
--import os, json
- import dlab.fab
- import dlab.common_lib
-import datalab.fab
-import datalab.common_lib
--import ast
class AzureActions:
@@@ -493,7 -493,7 +492,8 @@@
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
- def create_instance(self, region, instance_size, service_base_name, instance_name, dlab_ssh_user_name, public_key,
- def create_instance(self, region, instance_size, service_base_name, instance_name, datalab_ssh_user_name, public_key,
++ def create_instance(self, region, instance_size, service_base_name, instance_name, datalab_ssh_user_name,
++ public_key,
network_interface_resource_id, resource_group_name, primary_disk_size, instance_type,
image_full_name, tags, project_name='', create_option='fromImage', disk_id='',
instance_storage_account_type='Premium_LRS', image_type='default'):
@@@ -1177,8 -1177,8 +1177,8 @@@ def configure_dataengine_spark(cluster_
else:
local('cp -f /opt/hadoop/etc/hadoop/core-site.xml {}hadoop/etc/hadoop/core-site.xml'.format(cluster_dir))
if spark_configs and os.path.exists('{0}'.format(cluster_dir)):
- dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
- capture=True)
+ datalab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
- capture=True)
++ capture=True)
spark_configurations = ast.literal_eval(spark_configs)
new_spark_defaults = list()
spark_defaults = local('cat {0}spark/conf/spark-defaults.conf'.format(cluster_dir), capture=True)
diff --cc infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 26714f7,6e081e5..d4aa77d
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@@ -19,27 -19,27 +19,27 @@@
#
# ******************************************************************************
--from pprint import pprint
--from googleapiclient.discovery import build
--from google.cloud import exceptions
--from google.cloud import storage
--from googleapiclient import errors
++import ast
++import backoff
++import datalab.common_lib
++import datalab.fab
import google.auth
- from dlab.fab import *
-from datalab.fab import *
--import meta_lib
--import os
import json
import logging
++import meta_lib
++import os
++import random
++import sys
++import time
import traceback
--import sys, time
++import urllib2
from Crypto.PublicKey import RSA
++from datalab.fab import *
from fabric.api import *
--import urllib2
- import dlab.fab
- import dlab.common_lib
-import datalab.fab
-import datalab.common_lib
--import backoff
--import ast
--import random
++from google.cloud import exceptions
++from google.cloud import storage
++from googleapiclient import errors
++from googleapiclient.discovery import build
class GCPActions:
@@@ -1530,8 -1525,8 +1530,8 @@@ def configure_dataengine_spark(cluster_
cluster_dir))
local('cp -f /opt/spark/conf/core-site.xml {}spark/conf/'.format(cluster_dir))
if spark_configs and os.path.exists('{0}'.format(cluster_dir)):
- dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
- capture=True)
+ datalab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
- capture=True)
++ capture=True)
spark_configurations = ast.literal_eval(spark_configs)
new_spark_defaults = list()
spark_defaults = local('cat {0}spark/conf/spark-defaults.conf'.format(cluster_dir), capture=True)
diff --cc infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
index 6b7582b,104c9b4..7ea795e
--- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
@@@ -19,19 -19,19 +19,19 @@@
#
# ******************************************************************************
--from pprint import pprint
--from googleapiclient.discovery import build
--from google.cloud import storage
--from google.cloud import exceptions
++import backoff
import google.auth
- from dlab.fab import *
-from datalab.fab import *
--import actions_lib
--import os, re
--from googleapiclient import errors
import logging
++import os
++import re
++import sys
++import time
import traceback
--import sys, time
--import backoff
++from datalab.fab import *
++from google.cloud import exceptions
++from google.cloud import storage
++from googleapiclient import errors
++from googleapiclient.discovery import build
class GCPMeta:
diff --cc infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index 6b0f5d6,20e74c1..7d207c3
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@@ -23,12 -23,12 +23,12 @@@
import os
import sys
-from fabric.api import *
-from fabric.contrib.files import exists
-from datalab.common_lib import manage_pkg
++from datalab.common_lib import configure_nginx_LE
+ from datalab.common_lib import install_certbot
++from datalab.common_lib import manage_pkg
+ from datalab.common_lib import run_certbot
-from datalab.common_lib import configure_nginx_LE
+from fabric.api import *
+from fabric.contrib.files import exists
- from dlab.common_lib import manage_pkg
- from dlab.common_lib import install_certbot
- from dlab.common_lib import run_certbot
- from dlab.common_lib import configure_nginx_LE
def configure_http_proxy_server(config):
@@@ -84,9 -84,9 +84,9 @@@ def install_nginx_lua(edge_ip, nginx_ve
cn = edge_ip
sudo('step ca token {3} --kid {0} --ca-url "{1}" --root /etc/ssl/certs/root_ca.crt '
'--password-file /home/{2}/keys/provisioner_password {4} --output-file /tmp/step_token'.format(
-- os.environ['conf_stepcerts_kid'], os.environ['conf_stepcerts_ca_url'], user, cn, sans))
++ os.environ['conf_stepcerts_kid'], os.environ['conf_stepcerts_ca_url'], user, cn, sans))
token = sudo('cat /tmp/step_token')
- sudo('step ca certificate "{0}" /etc/ssl/certs/dlab.crt /etc/ssl/certs/dlab.key '
+ sudo('step ca certificate "{0}" /etc/ssl/certs/datalab.crt /etc/ssl/certs/datalab.key '
'--token "{1}" --kty=RSA --size 2048 --provisioner {2} '.format(cn, token,
os.environ['conf_stepcerts_kid']))
sudo('touch /var/log/renew_certificates.log')
@@@ -176,10 -176,10 +176,11 @@@ def configure_nftables(config)
elif os.environ['conf_cloud_provider'] == 'gcp':
interface = 'ens4'
sudo('sed -i \'s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g\' /etc/sysctl.conf')
- sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/dlab/templates/nftables.conf'.format(config['edge_ip']))
- sudo('sed -i "s|INTERFACE|{}|g" /opt/dlab/templates/nftables.conf'.format(interface))
- sudo('sed -i "s|SUBNET_CIDR|{}|g" /opt/dlab/templates/nftables.conf'.format(config['exploratory_subnet']))
- sudo('cp /opt/dlab/templates/nftables.conf /etc/')
+ sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/nftables.conf'.format(config['edge_ip']))
+ sudo('sed -i "s|INTERFACE|{}|g" /opt/datalab/templates/nftables.conf'.format(interface))
- sudo('sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet']))
++ sudo(
++ 'sed -i "s|SUBNET_CIDR|{}|g" /opt/datalab/templates/nftables.conf'.format(config['exploratory_subnet']))
+ sudo('cp /opt/datalab/templates/nftables.conf /etc/')
sudo('systemctl restart nftables')
sudo('touch /tmp/nftables_ensured')
except Exception as err:
diff --cc infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
index ea1bfd5,8370585..edda35e
--- a/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/notebook_lib.py
@@@ -21,19 -21,19 +21,15 @@@
#
# ******************************************************************************
--from fabric.api import *
--from fabric.contrib.files import exists
--import argparse
--import json
--import random
--import string
--import sys
- from dlab.notebook_lib import *
- from dlab.fab import *
- from dlab.common_lib import *
-from datalab.notebook_lib import *
-from datalab.fab import *
-from datalab.common_lib import *
import backoff
import os
import re
++import sys
++from datalab.common_lib import *
++from datalab.fab import *
++from datalab.notebook_lib import *
++from fabric.api import *
++from fabric.contrib.files import exists
def enable_proxy(proxy_host, proxy_port):
@@@ -125,8 -125,8 +121,9 @@@ def install_rstudio(os_user, local_spar
sudo('chown {0}:{0} /mnt/var'.format(os_user))
http_proxy = run('echo $http_proxy')
https_proxy = run('echo $https_proxy')
- sudo("sed -i '/Type=forking/a \Environment=USER=dlab-user' /etc/systemd/system/rstudio-server.service")
- sudo("sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service")
+ sudo("sed -i '/Type=forking/a \Environment=USER=datalab-user' /etc/systemd/system/rstudio-server.service")
- sudo("sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service")
++ sudo(
++ "sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service")
sudo("sed -i '/ExecStart/s|$|\"|g' /etc/systemd/system/rstudio-server.service")
sudo(
'sed -i \'/\[Service\]/a Environment=\"HTTP_PROXY={}\"\' /etc/systemd/system/rstudio-server.service'.format(
diff --cc infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index 0c38603,594ec74..ee53977
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@@ -21,21 -21,21 +21,20 @@@
#
# ******************************************************************************
--from fabric.api import *
import crypt
--import yaml
- from dlab.fab import *
- from dlab.meta_lib import *
-from datalab.fab import *
-from datalab.meta_lib import *
--import os
import json
--import traceback
++import os
import sys
- from dlab.common_lib import manage_pkg
++import traceback
+ from datalab.common_lib import manage_pkg
++from datalab.fab import *
++from datalab.meta_lib import *
++from fabric.api import *
- def ensure_docker_daemon(dlab_path, os_user, region):
+ def ensure_docker_daemon(datalab_path, os_user, region):
try:
- if not exists(dlab_path + 'tmp/docker_daemon_ensured'):
+ if not exists(datalab_path + 'tmp/docker_daemon_ensured'):
docker_version = os.environ['ssn_docker_version']
sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
@@@ -88,7 -88,7 +87,10 @@@ def configure_jenkins(datalab_path, os_
sudo('mkdir -p /var/lib/jenkins/jobs/')
sudo('chown -R ' + os_user + ':' + os_user + ' /var/lib/jenkins/')
put('/root/templates/jenkins_jobs/*', '/var/lib/jenkins/jobs/')
-- sudo("find /var/lib/jenkins/jobs/ -type f | xargs sed -i \'s/OS_USR/{}/g; s/SBN/{}/g; s/CTUN/{}/g; s/SGI/{}/g; s/VPC/{}/g; s/SNI/{}/g; s/AKEY/{}/g\'".format(os_user, config['service_base_name'], tag_resource_id, config['security_group_id'], config['vpc_id'], config['subnet_id'], config['admin_key']))
++ sudo(
++ "find /var/lib/jenkins/jobs/ -type f | xargs sed -i \'s/OS_USR/{}/g; s/SBN/{}/g; s/CTUN/{}/g; s/SGI/{}/g; s/VPC/{}/g; s/SNI/{}/g; s/AKEY/{}/g\'".format(
++ os_user, config['service_base_name'], tag_resource_id, config['security_group_id'],
++ config['vpc_id'], config['subnet_id'], config['admin_key']))
sudo('chown -R jenkins:jenkins /var/lib/jenkins')
sudo('/etc/init.d/jenkins stop; sleep 5')
sudo('systemctl enable jenkins')
@@@ -129,9 -129,9 +131,10 @@@ def configure_nginx(config, datalab_pat
with open(template_file) as tpl:
for line in tpl:
out.write(line)
-- put("/tmp/%s-tmpproxy_location_jenkins_template.conf" % random_file_part, '/tmp/proxy_location_jenkins.conf')
- sudo('mv /tmp/proxy_location_jenkins.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
- sudo('\cp ' + os.environ['ssn_dlab_path'] + 'tmp/proxy_location_jenkins.conf /etc/nginx/locations/')
++ put("/tmp/%s-tmpproxy_location_jenkins_template.conf" % random_file_part,
++ '/tmp/proxy_location_jenkins.conf')
+ sudo('mv /tmp/proxy_location_jenkins.conf ' + os.environ['ssn_datalab_path'] + 'tmp/')
+ sudo('\cp ' + os.environ['ssn_datalab_path'] + 'tmp/proxy_location_jenkins.conf /etc/nginx/locations/')
sudo("echo 'engineer:" + crypt.crypt(nginx_password, id_generator()) + "' > /etc/nginx/htpasswd")
with open('jenkins_creds.txt', 'w+') as f:
f.write("Jenkins credentials: engineer / " + nginx_password)
@@@ -173,10 -173,10 +176,11 @@@ def ensure_mongo()
print('Failed to install MongoDB: ', str(err))
sys.exit(1)
- def start_ss(keyfile, host_string, dlab_conf_dir, web_path,
++
+ def start_ss(keyfile, host_string, datalab_conf_dir, web_path,
os_user, mongo_passwd, keystore_passwd, cloud_provider,
service_base_name, tag_resource_id, billing_tag, account_id, billing_bucket,
- aws_job_enabled, dlab_path, billing_enabled, cloud_params,
+ aws_job_enabled, datalab_path, billing_enabled, cloud_params,
authentication_file, offer_number, currency,
locale, region_info, ldap_login, tenant_id,
application_id, hostname, data_lake_name, subscription_id,
@@@ -192,16 -192,16 +196,16 @@@
local('sed -i "s|CLOUD_PROVIDER|{}|g" /root/templates/ssn.yml'.format(cloud_provider))
local('sed -i "s|\${JRE_HOME}|' + java_path + '|g" /root/templates/ssn.yml')
sudo('sed -i "s|KEYNAME|{}|g" {}/webapp/provisioning-service/conf/provisioning.yml'.
- format(os.environ['conf_key_name'], dlab_path))
+ format(os.environ['conf_key_name'], datalab_path))
put('/root/templates/ssn.yml', '/tmp/ssn.yml')
- sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
+ sudo('mv /tmp/ssn.yml ' + os.environ['ssn_datalab_path'] + 'conf/')
put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
- sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
+ sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_datalab_path'] + 'tmp/')
if cloud_provider == 'aws':
- conf_parameter_name = '--spring.config.location={0}billing_app.yml --conf '.format(dlab_conf_dir)
+ conf_parameter_name = '--spring.config.location={0}billing_app.yml --conf '.format(datalab_conf_dir)
with open('/root/templates/supervisor_svc.conf', 'r') as f:
text = f.read()
- text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
- text = text.replace('WEB_CONF', datalab_conf_dir).replace('OS_USR', os_user)\
++ text = text.replace('WEB_CONF', datalab_conf_dir).replace('OS_USR', os_user) \
.replace('CONF_PARAMETER_NAME', conf_parameter_name)
with open('/root/templates/supervisor_svc.conf', 'w') as f:
f.write(text)
@@@ -209,7 -209,7 +213,7 @@@
conf_parameter_name = '--spring.config.location='
with open('/root/templates/supervisor_svc.conf', 'r') as f:
text = f.read()
- text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
- text = text.replace('WEB_CONF', datalab_conf_dir).replace('OS_USR', os_user)\
++ text = text.replace('WEB_CONF', datalab_conf_dir).replace('OS_USR', os_user) \
.replace('CONF_PARAMETER_NAME', conf_parameter_name)
with open('/root/templates/supervisor_svc.conf', 'w') as f:
f.write(text)
@@@ -241,7 -241,7 +245,8 @@@
sudo('sed -i "s|<LOGIN_USE_LDAP>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(ldap_login))
sudo('sed -i "s|<LOGIN_TENANT_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(tenant_id))
sudo('sed -i "s|<LOGIN_APPLICATION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(application_id))
- sudo('sed -i "s|<DLAB_SUBSCRIPTION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(subscription_id))
- sudo('sed -i "s|<DATALAB_SUBSCRIPTION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(subscription_id))
++ sudo('sed -i "s|<DATALAB_SUBSCRIPTION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(
++ subscription_id))
sudo('sed -i "s|<MANAGEMENT_API_AUTH_FILE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(
authentication_file))
sudo('sed -i "s|<VALIDATE_PERMISSION_SCOPE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(
@@@ -287,7 -287,7 +292,7 @@@
'--cost {} ' \
'--resource_id {} ' \
'--tags {} ' \
-- '--billing_dataset_name "{}" '\
++ '--billing_dataset_name "{}" ' \
'--mongo_host localhost ' \
'--mongo_port 27017 ' \
'--service_base_name {} ' \
@@@ -329,11 -329,11 +334,12 @@@
sudo('python /tmp/configure_billing.py {}'.format(params))
try:
if os.environ['conf_stepcerts_enabled'] == 'true':
- sudo('openssl pkcs12 -export -in /etc/ssl/certs/dlab.crt -inkey /etc/ssl/certs/dlab.key -name ssn '
- sudo('openssl pkcs12 -export -in /etc/ssl/certs/datalab.crt -inkey /etc/ssl/certs/datalab.key -name ssn '
-- '-out ssn.p12 -password pass:{0}'.format(keystore_passwd))
++ sudo(
++ 'openssl pkcs12 -export -in /etc/ssl/certs/datalab.crt -inkey /etc/ssl/certs/datalab.key -name ssn '
++ '-out ssn.p12 -password pass:{0}'.format(keystore_passwd))
sudo('keytool -importkeystore -srckeystore ssn.p12 -srcstoretype PKCS12 -alias ssn -destkeystore '
'/home/{0}/keys/ssn.keystore.jks -deststorepass "{1}" -srcstorepass "{1}"'.format(
-- os_user, keystore_passwd))
++ os_user, keystore_passwd))
sudo('keytool -keystore /home/{0}/keys/ssn.keystore.jks -alias step-ca -import -file '
'/etc/ssl/certs/root_ca.crt -deststorepass "{1}" -srcstorepass "{1}" -noprompt'.format(
os_user, keystore_passwd))
@@@ -344,10 -344,10 +350,10 @@@
else:
sudo('keytool -genkeypair -alias ssn -keyalg RSA -validity 730 -storepass {1} -keypass {1} \
-keystore /home/{0}/keys/ssn.keystore.jks -keysize 2048 -dname "CN=localhost"'.format(
-- os_user, keystore_passwd))
- sudo('keytool -exportcert -alias ssn -storepass {1} -file /etc/ssl/certs/dlab.crt \
++ os_user, keystore_passwd))
+ sudo('keytool -exportcert -alias ssn -storepass {1} -file /etc/ssl/certs/datalab.crt \
-keystore /home/{0}/keys/ssn.keystore.jks'.format(os_user, keystore_passwd))
- sudo('keytool -importcert -trustcacerts -alias ssn -file /etc/ssl/certs/dlab.crt -noprompt \
+ sudo('keytool -importcert -trustcacerts -alias ssn -file /etc/ssl/certs/datalab.crt -noprompt \
-storepass changeit -keystore {1}/lib/security/cacerts'.format(os_user, java_path))
except:
append_result("Unable to generate cert and copy to java keystore")
@@@ -368,8 -368,8 +374,9 @@@ def install_build_dep()
maven_version = '3.5.4'
manage_pkg('-y install', 'remote', 'openjdk-8-jdk git wget unzip')
with cd('/opt/'):
-- sudo('wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
-- maven_version.split('.')[0], maven_version))
++ sudo(
++ 'wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
++ maven_version.split('.')[0], maven_version))
sudo('unzip apache-maven-{}-bin.zip'.format(maven_version))
sudo('mv apache-maven-{} maven'.format(maven_version))
sudo('bash -c "curl --silent --location https://deb.nodesource.com/setup_12.x | bash -"')
diff --cc infrastructure-provisioning/src/general/lib/os/fab.py
index 6758c1f,f8d1dfb..0d99831
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@@ -19,20 -19,20 +19,22 @@@
#
# ******************************************************************************
--from fabric.api import *
--from fabric.contrib.files import exists
++import csv
++import datetime
++import json
import logging
import os
import random
--import sys
--import string
--import json, uuid, time, datetime, csv
- from dlab.meta_lib import *
- from dlab.actions_lib import *
- import dlab.actions_lib
-from datalab.meta_lib import *
-from datalab.actions_lib import *
-import datalab.actions_lib
import re
++import string
++import sys
++import time
import traceback
- from dlab.common_lib import *
++from datalab.actions_lib import *
+ from datalab.common_lib import *
++from datalab.meta_lib import *
++from fabric.api import *
++from fabric.contrib.files import exists
def ensure_pip(requisites):
@@@ -295,24 -295,24 +297,26 @@@ def ensure_jupyterlab_files(os_user, ju
sudo('echo \'c.NotebookApp.cookie_secret = b"{0}"\' >> {1}'.format(id_generator(), jupyter_conf_file))
sudo('''echo "c.NotebookApp.token = u''" >> {}'''.format(jupyter_conf_file))
sudo('echo \'c.KernelSpecManager.ensure_native_kernel = False\' >> {}'.format(jupyter_conf_file))
- sudo('chown dlab-user:dlab-user /opt')
- sudo('echo -e "Host git.epam.com\n HostName git.epam.com\n ProxyCommand nc -X connect -x {}:3128 %h %p\n" > /home/{}/.ssh/config'.format(edge_ip, os_user))
+ sudo('chown datalab-user:datalab-user /opt')
- sudo('echo -e "Host git.epam.com\n HostName git.epam.com\n ProxyCommand nc -X connect -x {}:3128 %h %p\n" > /home/{}/.ssh/config'.format(edge_ip, os_user))
++ sudo(
++ 'echo -e "Host git.epam.com\n HostName git.epam.com\n ProxyCommand nc -X connect -x {}:3128 %h %p\n" > /home/{}/.ssh/config'.format(
++ edge_ip, os_user))
sudo('echo -e "Host github.com\n HostName github.com\n ProxyCommand nc -X connect -x {}:3128 %h %p" >> /home/{}/.ssh/config'.format(edge_ip, os_user))
# sudo('touch {}'.format(spark_script))
# sudo('echo "#!/bin/bash" >> {}'.format(spark_script))
# sudo(
# 'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/pyspark_local_template.json" >> {}'.format(
# spark_script))
--# sudo(
- # 'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/pyspark_local_template.json" >> {}'.format(
-# 'echo "sed -i \'14s/:",/:\\/home\\/datalab-user\\/caffe\\/python:\\/home\\/datalab-user\\/pytorch\\/build:",/\' /tmp/pyspark_local_template.json" >> {}'.format(
--# spark_script))
++ # sudo(
++ # 'echo "sed -i \'14s/:",/:\\/home\\/datalab-user\\/caffe\\/python:\\/home\\/datalab-user\\/pytorch\\/build:",/\' /tmp/pyspark_local_template.json" >> {}'.format(
++ # spark_script))
# sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/pyspark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
# sudo(
# 'echo "PYJ=\`find /opt/spark/ -name \'*py4j*.zip\' | tr \'\\n\' \':\' | sed \'s|:$||g\'\`; sed -i \'s|PY4J|\'$PYJ\'|g\' /tmp/py3spark_local_template.json" >> {}'.format(
# spark_script))
--# sudo(
- # 'echo "sed -i \'14s/:",/:\\/home\\/dlab-user\\/caffe\\/python:\\/home\\/dlab-user\\/pytorch\\/build:",/\' /tmp/py3spark_local_template.json" >> {}'.format(
-# 'echo "sed -i \'14s/:",/:\\/home\\/datalab-user\\/caffe\\/python:\\/home\\/datalab-user\\/pytorch\\/build:",/\' /tmp/py3spark_local_template.json" >> {}'.format(
--# spark_script))
++ # sudo(
++ # 'echo "sed -i \'14s/:",/:\\/home\\/datalab-user\\/caffe\\/python:\\/home\\/datalab-user\\/pytorch\\/build:",/\' /tmp/py3spark_local_template.json" >> {}'.format(
++ # spark_script))
# sudo('echo \'sed -i "s|SP_VER|{}|g" /tmp/py3spark_local_template.json\' >> {}'.format(os.environ['notebook_spark_version'], spark_script))
# sudo('echo "cp /tmp/pyspark_local_template.json /home/{}/.local/share/jupyter/kernels/pyspark_local/kernel.json" >> {}'.format(os_user, spark_script))
# sudo(
@@@ -946,29 -946,29 +950,41 @@@ def configure_superset(os_user, keycloa
try:
if not exists('/home/{}/incubator-superset'.format(os_user)):
with cd('/home/{}'.format(os_user)):
-- sudo('wget https://github.com/apache/incubator-superset/archive/{}.tar.gz'.format(os.environ['notebook_superset_version']))
++ sudo('wget https://github.com/apache/incubator-superset/archive/{}.tar.gz'.format(
++ os.environ['notebook_superset_version']))
sudo('tar -xzf {}.tar.gz'.format(os.environ['notebook_superset_version']))
sudo('ln -sf incubator-superset-{} incubator-superset'.format(os.environ['notebook_superset_version']))
if not exists('/tmp/superset-notebook_installed'):
- sudo('mkdir -p /opt/dlab/templates')
- put('/root/templates', '/opt/dlab', use_sudo=True)
- sudo('sed -i \'s/OS_USER/{}/g\' /opt/dlab/templates/.env'.format(os_user))
+ sudo('mkdir -p /opt/datalab/templates')
+ put('/root/templates', '/opt/datalab', use_sudo=True)
+ sudo('sed -i \'s/OS_USER/{}/g\' /opt/datalab/templates/.env'.format(os_user))
proxy_string = '{}:3128'.format(edge_instance_private_ip)
- sudo('sed -i \'s|KEYCLOAK_AUTH_SERVER_URL|{}|g\' /opt/dlab/templates/id_provider.json'.format(keycloak_auth_server_url))
- sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/dlab/templates/id_provider.json'.format(keycloak_realm_name))
- sudo('sed -i \'s/CLIENT_ID/{}/g\' /opt/dlab/templates/id_provider.json'.format(keycloak_client_id))
- sudo('sed -i \'s/CLIENT_SECRET/{}/g\' /opt/dlab/templates/id_provider.json'.format(keycloak_client_secret))
- sudo('sed -i \'s/PROXY_STRING/{}/g\' /opt/dlab/templates/docker-compose.yml'.format(proxy_string))
- sudo('sed -i \'s|KEYCLOAK_AUTH_SERVER_URL|{}|g\' /opt/dlab/templates/superset_config.py'.format(keycloak_auth_server_url))
- sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/dlab/templates/superset_config.py'.format(keycloak_realm_name))
- sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/dlab/templates/superset_config.py'.format(edge_instance_public_ip))
- sudo('sed -i \'s/SUPERSET_NAME/{}/g\' /opt/dlab/templates/superset_config.py'.format(superset_name))
- sudo('cp -f /opt/dlab/templates/.env /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/dlab/templates/docker-compose.yml /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/dlab/templates/id_provider.json /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/dlab/templates/requirements-extra.txt /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/dlab/templates/superset_config.py /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/dlab/templates/docker-init.sh /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('sed -i \'s|KEYCLOAK_AUTH_SERVER_URL|{}|g\' /opt/datalab/templates/id_provider.json'.format(keycloak_auth_server_url))
- sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/id_provider.json'.format(keycloak_realm_name))
++ sudo('sed -i \'s|KEYCLOAK_AUTH_SERVER_URL|{}|g\' /opt/datalab/templates/id_provider.json'.format(
++ keycloak_auth_server_url))
++ sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/id_provider.json'.format(
++ keycloak_realm_name))
+ sudo('sed -i \'s/CLIENT_ID/{}/g\' /opt/datalab/templates/id_provider.json'.format(keycloak_client_id))
- sudo('sed -i \'s/CLIENT_SECRET/{}/g\' /opt/datalab/templates/id_provider.json'.format(keycloak_client_secret))
++ sudo('sed -i \'s/CLIENT_SECRET/{}/g\' /opt/datalab/templates/id_provider.json'.format(
++ keycloak_client_secret))
+ sudo('sed -i \'s/PROXY_STRING/{}/g\' /opt/datalab/templates/docker-compose.yml'.format(proxy_string))
- sudo('sed -i \'s|KEYCLOAK_AUTH_SERVER_URL|{}|g\' /opt/datalab/templates/superset_config.py'.format(keycloak_auth_server_url))
- sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/superset_config.py'.format(keycloak_realm_name))
++ sudo('sed -i \'s|KEYCLOAK_AUTH_SERVER_URL|{}|g\' /opt/datalab/templates/superset_config.py'.format(
++ keycloak_auth_server_url))
++ sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/superset_config.py'.format(
++ keycloak_realm_name))
+ sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/superset_config.py'.format(edge_instance_public_ip))
+ sudo('sed -i \'s/SUPERSET_NAME/{}/g\' /opt/datalab/templates/superset_config.py'.format(superset_name))
+ sudo('cp -f /opt/datalab/templates/.env /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/datalab/templates/docker-compose.yml /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/datalab/templates/id_provider.json /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/datalab/templates/requirements-extra.txt /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/datalab/templates/superset_config.py /home/{}/incubator-superset/contrib/docker/'.format(os_user))
- sudo('cp -f /opt/datalab/templates/docker-init.sh /home/{}/incubator-superset/contrib/docker/'.format(os_user))
++ sudo('cp -f /opt/datalab/templates/docker-compose.yml /home/{}/incubator-superset/contrib/docker/'.format(
++ os_user))
++ sudo('cp -f /opt/datalab/templates/id_provider.json /home/{}/incubator-superset/contrib/docker/'.format(
++ os_user))
++ sudo(
++ 'cp -f /opt/datalab/templates/requirements-extra.txt /home/{}/incubator-superset/contrib/docker/'.format(
++ os_user))
++ sudo('cp -f /opt/datalab/templates/superset_config.py /home/{}/incubator-superset/contrib/docker/'.format(
++ os_user))
++ sudo('cp -f /opt/datalab/templates/docker-init.sh /home/{}/incubator-superset/contrib/docker/'.format(
++ os_user))
sudo('touch /tmp/superset-notebook_installed')
except Exception as err:
print("Failed configure superset: " + str(err))
diff --cc infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
index 3ee832e,d58232d..27f346c
--- a/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/edge_lib.py
@@@ -23,9 -23,9 +23,9 @@@
import os
import sys
++from datalab.common_lib import manage_pkg
from fabric.api import *
from fabric.contrib.files import exists
- from dlab.common_lib import manage_pkg
-from datalab.common_lib import manage_pkg
def configure_http_proxy_server(config):
@@@ -83,9 -83,9 +83,9 @@@ def install_nginx_lua(edge_ip, nginx_ve
cn = edge_ip
sudo('step ca token {3} --kid {0} --ca-url "{1}" --root /etc/ssl/certs/root_ca.crt '
'--password-file /home/{2}/keys/provisioner_password {4} --output-file /tmp/step_token'.format(
-- os.environ['conf_stepcerts_kid'], os.environ['conf_stepcerts_ca_url'], user, cn, sans))
++ os.environ['conf_stepcerts_kid'], os.environ['conf_stepcerts_ca_url'], user, cn, sans))
token = sudo('cat /tmp/step_token')
- sudo('step ca certificate "{0}" /etc/ssl/certs/dlab.crt /etc/ssl/certs/dlab.key '
+ sudo('step ca certificate "{0}" /etc/ssl/certs/datalab.crt /etc/ssl/certs/datalab.key '
'--token "{1}" --kty=RSA --size 2048 --provisioner {2} '.format(cn, token,
os.environ['conf_stepcerts_kid']))
sudo('touch /var/log/renew_certificates.log')
@@@ -112,9 -112,9 +112,10 @@@
sudo('systemctl enable step-cert-manager.service')
else:
if os.environ['conf_letsencrypt_enabled'] == 'true':
- print('Lets Encrypt certificates are not supported for redhat in dlab. Using self signed certificates')
- sudo('openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/dlab.key \
- -out /etc/ssl/certs/dlab.crt -subj "/C=US/ST=US/L=US/O=dlab/CN={}"'.format(hostname))
- print('Lets Encrypt certificates are not supported for redhat in DataLab. Using self signed certificates')
++ print(
++ 'Lets Encrypt certificates are not supported for redhat in DataLab. Using self signed certificates')
+ sudo('openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/datalab.key \
+ -out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN={}"'.format(hostname))
sudo('mkdir -p /tmp/lua')
sudo('mkdir -p /tmp/src')
with cd('/tmp/src/'):
@@@ -166,19 -166,19 +167,23 @@@
sudo('useradd -r nginx')
sudo('rm -f /etc/nginx/nginx.conf')
- sudo('mkdir -p /opt/dlab/templates')
- put('/root/templates', '/opt/dlab', use_sudo=True)
- sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/dlab/templates/conf.d/proxy.conf'.format(edge_ip))
- sudo('sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/dlab/templates/conf.d/proxy.conf'.format(keycloak_auth_server_url))
- sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/dlab/templates/conf.d/proxy.conf'.format(keycloak_realm_name))
- sudo('sed -i \'s/KEYCLOAK_CLIENT_ID/{}/g\' /opt/dlab/templates/conf.d/proxy.conf'.format(keycloak_client_id))
- sudo('sed -i \'s/KEYCLOAK_CLIENT_SECRET/{}/g\' /opt/dlab/templates/conf.d/proxy.conf'.format(keycloak_client_secret))
-
- sudo('cp /opt/dlab/templates/nginx.conf /etc/nginx/')
+ sudo('mkdir -p /opt/datalab/templates')
+ put('/root/templates', '/opt/datalab', use_sudo=True)
+ sudo('sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(edge_ip))
- sudo('sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/datalab/templates/conf.d/proxy.conf'.format(keycloak_auth_server_url))
- sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(keycloak_realm_name))
- sudo('sed -i \'s/KEYCLOAK_CLIENT_ID/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(keycloak_client_id))
- sudo('sed -i \'s/KEYCLOAK_CLIENT_SECRET/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(keycloak_client_secret))
++ sudo('sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
++ keycloak_auth_server_url))
++ sudo('sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
++ keycloak_realm_name))
++ sudo('sed -i \'s/KEYCLOAK_CLIENT_ID/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
++ keycloak_client_id))
++ sudo('sed -i \'s/KEYCLOAK_CLIENT_SECRET/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'.format(
++ keycloak_client_secret))
+
+ sudo('cp /opt/datalab/templates/nginx.conf /etc/nginx/')
sudo('mkdir /etc/nginx/conf.d')
- sudo('cp /opt/dlab/templates/conf.d/proxy.conf /etc/nginx/conf.d/')
+ sudo('cp /opt/datalab/templates/conf.d/proxy.conf /etc/nginx/conf.d/')
sudo('mkdir /etc/nginx/locations')
- sudo('cp /opt/dlab/templates/nginx_redhat /etc/init.d/nginx')
+ sudo('cp /opt/datalab/templates/nginx_redhat /etc/init.d/nginx')
sudo('chmod +x /etc/init.d/nginx')
sudo('chkconfig --add nginx')
sudo('chkconfig --level 345 nginx on')
diff --cc infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
index 1cee42e,5f6b7e7..284c941
--- a/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/notebook_lib.py
@@@ -21,17 -21,17 +21,14 @@@
#
# ******************************************************************************
--from fabric.api import *
--from fabric.contrib.files import exists
--import argparse
import json
--import random
--import string
++import os
import sys
- from dlab.notebook_lib import *
- from dlab.fab import *
-from datalab.notebook_lib import *
-from datalab.fab import *
--import os, time
- from dlab.common_lib import manage_pkg
+ from datalab.common_lib import manage_pkg
++from datalab.fab import *
++from datalab.notebook_lib import *
++from fabric.api import *
++from fabric.contrib.files import exists
def enable_proxy(proxy_host, proxy_port):
@@@ -112,8 -112,8 +109,9 @@@ def install_rstudio(os_user, local_spar
manage_pkg('-y install --nogpgcheck', 'remote', 'https://download2.rstudio.org/server/centos6/x86_64/rstudio-server-rhel-{}-x86_64.rpm'.format(rstudio_version))
sudo('mkdir -p /mnt/var')
sudo('chown {0}:{0} /mnt/var'.format(os_user))
- sudo("sed -i '/Type=forking/a \Environment=USER=dlab-user' /etc/systemd/system/rstudio-server.service")
- sudo("sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service")
+ sudo("sed -i '/Type=forking/a \Environment=USER=datalab-user' /etc/systemd/system/rstudio-server.service")
- sudo("sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service")
++ sudo(
++ "sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service")
sudo("sed -i '/ExecStart/s|$|\"|g' /etc/systemd/system/rstudio-server.service")
sudo("systemctl daemon-reload")
sudo('touch /home/{}/.Renviron'.format(os_user))
diff --cc infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
index a0022da,775576e..8a2c357
--- a/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/ssn_lib.py
@@@ -21,21 -21,21 +21,20 @@@
#
# ******************************************************************************
--from fabric.api import *
import crypt
--import yaml
- from dlab.fab import *
- from dlab.meta_lib import *
-from datalab.fab import *
-from datalab.meta_lib import *
--import os
import json
++import os
import sys
import traceback
- from dlab.common_lib import manage_pkg
+ from datalab.common_lib import manage_pkg
++from datalab.fab import *
++from datalab.meta_lib import *
++from fabric.api import *
- def ensure_docker_daemon(dlab_path, os_user, region):
+ def ensure_docker_daemon(datalab_path, os_user, region):
try:
- if not exists('{}tmp/docker_daemon_ensured'.format(dlab_path)):
+ if not exists('{}tmp/docker_daemon_ensured'.format(datalab_path)):
docker_version = os.environ['ssn_docker_version']
if region == 'cn-north-1':
mirror = 'mirror.lzu.edu.cn'
@@@ -98,11 -98,11 +97,15 @@@ def configure_jenkins(datalab_path, os_
sudo('mkdir -p /var/lib/jenkins/jobs/')
sudo('chown -R {0}:{0} /var/lib/jenkins/'.format(os_user))
put('/root/templates/jenkins_jobs/*', '/var/lib/jenkins/jobs/')
-- #sudo("find /var/lib/jenkins/jobs/ -type f | xargs sed -i \'s/OS_USR/{}/g\'".format(os_user))
-- sudo("find /var/lib/jenkins/jobs/ -type f | xargs sed -i \'s/OS_USR/{}/g; s/SBN/{}/g; s/CTUN/{}/g; s/SGI/{}/g; s/VPC/{}/g; s/SNI/{}/g; s/AKEY/{}/g\'".format(os_user, config['service_base_name'], tag_resource_id, config['security_group_id'], config['vpc_id'], config['subnet_id'], config['admin_key']))
++ # sudo("find /var/lib/jenkins/jobs/ -type f | xargs sed -i \'s/OS_USR/{}/g\'".format(os_user))
++ sudo(
++ "find /var/lib/jenkins/jobs/ -type f | xargs sed -i \'s/OS_USR/{}/g; s/SBN/{}/g; s/CTUN/{}/g; s/SGI/{}/g; s/VPC/{}/g; s/SNI/{}/g; s/AKEY/{}/g\'".format(
++ os_user, config['service_base_name'], tag_resource_id, config['security_group_id'],
++ config['vpc_id'], config['subnet_id'], config['admin_key']))
sudo('chown -R jenkins:jenkins /var/lib/jenkins')
sudo('/etc/init.d/jenkins stop; sleep 5')
-- sudo('sed -i \'/JENKINS_PORT/ s/^/#/\' /etc/sysconfig/jenkins; echo \'JENKINS_PORT="8070"\' >> /etc/sysconfig/jenkins')
++ sudo(
++ 'sed -i \'/JENKINS_PORT/ s/^/#/\' /etc/sysconfig/jenkins; echo \'JENKINS_PORT="8070"\' >> /etc/sysconfig/jenkins')
sudo('sed -i \'/JENKINS_ARGS/ s|=""|="--prefix=/jenkins"|\' /etc/sysconfig/jenkins')
sudo('semanage port -a -t http_port_t -p tcp 8070')
sudo('setsebool -P httpd_can_network_connect 1')
@@@ -160,12 -160,12 +163,12 @@@ def configure_nginx(config, datalab_pat
def ensure_supervisor():
try:
- if not exists('{}tmp/superv_ensured'.format(os.environ['ssn_dlab_path'])):
+ if not exists('{}tmp/superv_ensured'.format(os.environ['ssn_datalab_path'])):
manage_pkg('-y install', 'remote', 'supervisor')
-- #sudo('pip install supervisor')
++ # sudo('pip install supervisor')
sudo('chkconfig supervisord on')
sudo('systemctl start supervisord')
- sudo('touch {}tmp/superv_ensured'.format(os.environ['ssn_dlab_path']))
+ sudo('touch {}tmp/superv_ensured'.format(os.environ['ssn_datalab_path']))
except Exception as err:
traceback.print_exc()
print('Failed to install supervisor: ', str(err))
@@@ -213,16 -213,16 +216,16 @@@ def start_ss(keyfile, host_string, data
local('sed -i "s|CLOUD_PROVIDER|{}|g" /root/templates/ssn.yml'.format(cloud_provider))
local('sed -i "s|\${JRE_HOME}|' + java_path + '|g" /root/templates/ssn.yml')
sudo('sed -i "s|KEYNAME|{}|g" {}/webapp/provisioning-service/conf/provisioning.yml'.
- format(os.environ['conf_key_name'], dlab_path))
- format(os.environ['conf_key_name'], datalab_path))
++ format(os.environ['conf_key_name'], datalab_path))
put('/root/templates/ssn.yml', '/tmp/ssn.yml')
- sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
+ sudo('mv /tmp/ssn.yml ' + os.environ['ssn_datalab_path'] + 'conf/')
put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
- sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
+ sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_datalab_path'] + 'tmp/')
if cloud_provider == 'gcp':
conf_parameter_name = '--spring.config.location='
with open('/root/templates/supervisor_svc.conf', 'r') as f:
text = f.read()
- text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
- text = text.replace('WEB_CONF', datalab_conf_dir).replace('OS_USR', os_user)\
++ text = text.replace('WEB_CONF', datalab_conf_dir).replace('OS_USR', os_user) \
.replace('CONF_PARAMETER_NAME', conf_parameter_name)
with open('/root/templates/supervisor_svc.conf', 'w') as f:
f.write(text)
@@@ -230,7 -230,7 +233,7 @@@
conf_parameter_name = '--conf '
with open('/root/templates/supervisor_svc.conf', 'r') as f:
text = f.read()
- text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
- text = text.replace('WEB_CONF', datalab_conf_dir).replace('OS_USR', os_user)\
++ text = text.replace('WEB_CONF', datalab_conf_dir).replace('OS_USR', os_user) \
.replace('CONF_PARAMETER_NAME', conf_parameter_name)
with open('/root/templates/supervisor_svc.conf', 'w') as f:
f.write(text)
@@@ -262,8 -262,8 +265,10 @@@
sudo('sed -i "s|<LOGIN_USE_LDAP>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(ldap_login))
sudo('sed -i "s|<LOGIN_TENANT_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(tenant_id))
sudo('sed -i "s|<LOGIN_APPLICATION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(application_id))
- sudo('sed -i "s|<DLAB_SUBSCRIPTION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(subscription_id))
- sudo('sed -i "s|<DATALAB_SUBSCRIPTION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(subscription_id))
-- sudo('sed -i "s|<MANAGEMENT_API_AUTH_FILE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(authentication_file))
++ sudo('sed -i "s|<DATALAB_SUBSCRIPTION_ID>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(
++ subscription_id))
++ sudo('sed -i "s|<MANAGEMENT_API_AUTH_FILE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(
++ authentication_file))
sudo('sed -i "s|<VALIDATE_PERMISSION_SCOPE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(validate_permission_scope))
sudo('sed -i "s|<LOGIN_APPLICATION_REDIRECT_URL>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(hostname))
sudo('sed -i "s|<LOGIN_PAGE>|{0}|g" /tmp/yml_tmp/self-service.yml'.format(hostname))
@@@ -343,8 -343,8 +348,9 @@@
try:
if os.environ['conf_stepcerts_enabled'] == 'true':
- sudo('openssl pkcs12 -export -in /etc/ssl/certs/dlab.crt -inkey /etc/ssl/certs/dlab.key -name ssn '
- sudo('openssl pkcs12 -export -in /etc/ssl/certs/datalab.crt -inkey /etc/ssl/certs/datalab.key -name ssn '
-- '-out ssn.p12 -password pass:{0}'.format(keystore_passwd))
++ sudo(
++ 'openssl pkcs12 -export -in /etc/ssl/certs/datalab.crt -inkey /etc/ssl/certs/datalab.key -name ssn '
++ '-out ssn.p12 -password pass:{0}'.format(keystore_passwd))
sudo('keytool -importkeystore -srckeystore ssn.p12 -srcstoretype PKCS12 -alias ssn -destkeystore '
'/home/{0}/keys/ssn.keystore.jks -deststorepass "{1}" -srcstorepass "{1}"'.format(
os_user, keystore_passwd))
@@@ -357,7 -357,7 +363,8 @@@
'-storepass changeit -keystore {0}/lib/security/cacerts'.format(java_path))
else:
if os.environ['conf_letsencrypt_enabled'] == 'true':
- print('Lets Encrypt certificates are not supported for redhat in dlab. Using self signed certificates')
- print('Lets Encrypt certificates are not supported for redhat in datalab. Using self signed certificates')
++ print(
++ 'Lets Encrypt certificates are not supported for redhat in datalab. Using self signed certificates')
sudo('keytool -genkeypair -alias ssn -keyalg RSA -validity 730 -storepass {1} -keypass {1} \
-keystore /home/{0}/keys/ssn.keystore.jks -keysize 2048 -dname "CN=localhost"'.format(
os_user, keystore_passwd))
@@@ -383,8 -383,8 +390,9 @@@ def install_build_dep()
maven_version = '3.5.4'
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk java-1.8.0-openjdk-devel git wget unzip')
with cd('/opt/'):
-- sudo('wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
-- maven_version.split('.')[0], maven_version))
++ sudo(
++ 'wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
++ maven_version.split('.')[0], maven_version))
sudo('unzip apache-maven-{}-bin.zip'.format(maven_version))
sudo('mv apache-maven-{} maven'.format(maven_version))
sudo('bash -c "curl --silent --location https://rpm.nodesource.com/setup_12.x | bash -"')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
index e63a63f,fe62e8c..d20f06b
--- a/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_collect_data.py
@@@ -22,22 -22,22 +22,19 @@@
# ******************************************************************************
import argparse
++import ast
import json
--import datetime
--from fabric.api import *
- from dlab.actions_lib import *
- from dlab.meta_lib import *
- from dlab.fab import *
- import traceback
+import sys
- import ast
-
++import traceback
+ from datalab.actions_lib import *
-from datalab.meta_lib import *
+ from datalab.fab import *
-import traceback
-import sys
-import ast
-
++from datalab.meta_lib import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--list_resources', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
data = ast.literal_eval(args.list_resources.replace('\'', '"'))
statuses = {}
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
index 207af06,6a09b96..22434f0
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
@@@ -22,11 -22,11 +22,9 @@@
# ******************************************************************************
import argparse
--import json
- from dlab.actions_lib import *
- from dlab.meta_lib import *
+import sys
-
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
-
parser = argparse.ArgumentParser()
parser.add_argument('--bucket_name', type=str, default='')
@@@ -35,7 -35,7 +33,6 @@@ parser.add_argument('--region', type=st
parser.add_argument('--bucket_name_tag', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
if args.bucket_name != '':
try:
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_create_instance.py
index ecf46be,a97f228..d5971fd
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_instance.py
@@@ -23,10 -23,10 +23,9 @@@
import argparse
import json
- from dlab.actions_lib import *
- from dlab.meta_lib import *
+import sys
-
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
-
parser = argparse.ArgumentParser()
parser.add_argument('--node_name', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_create_notebook_image.py
index 1d5cb04,39481bc..7cfa0cf
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_notebook_image.py
@@@ -21,14 -21,14 +21,10 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import sys
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import sys
import json
--import uuid
import os
--
++import sys
if __name__ == "__main__":
try:
@@@ -64,10 -64,10 +60,10 @@@
except KeyError:
os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(
os.environ['project_name'], os.environ['endpoint_name'])
- image_id = dlab.actions_lib.create_image_from_instance(tag_name=image_conf['instance_tag'],
- instance_name=image_conf['instance_name'],
- image_name=image_conf['full_image_name'],
- tags=json.dumps(image_conf['tags']))
+ image_id = datalab.actions_lib.create_image_from_instance(tag_name=image_conf['instance_tag'],
- instance_name=image_conf['instance_name'],
- image_name=image_conf['full_image_name'],
- tags=json.dumps(image_conf['tags']))
++ instance_name=image_conf['instance_name'],
++ image_name=image_conf['full_image_name'],
++ tags=json.dumps(image_conf['tags']))
print("Image was successfully created. It's name is {}".format(image_conf['full_image_name']))
with open("/root/result.json", 'w') as result:
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
index 8b7f038,1bef0c6..b804e8a
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_policy.py
@@@ -22,10 -22,10 +22,11 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
++import boto3
++import botocore
+import sys
- import boto3, botocore
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
-import boto3, botocore
parser = argparse.ArgumentParser()
parser.add_argument('--bucket_name', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
index 1f914c1,4763bd3..c3eae52
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_role_policy.py
@@@ -22,9 -22,9 +22,9 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
+import sys
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
parser = argparse.ArgumentParser()
parser.add_argument('--role_name', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_create_security_group.py
index 4ee0575,3fd252f..f06bf41
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_security_group.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_security_group.py
@@@ -21,13 -21,13 +21,11 @@@
#
# ******************************************************************************
--import json
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
++import json
+import sys
- from botocore.exceptions import ClientError
-
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
-from botocore.exceptions import ClientError
-
parser = argparse.ArgumentParser()
parser.add_argument('--name', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
index 83dd48a,d5b560b..6bc4598
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_subnet.py
@@@ -22,14 -22,14 +22,13 @@@
# ******************************************************************************
import argparse
- import json
- from botocore import exceptions
- from dlab.actions_lib import *
- from dlab.meta_lib import *
- import sys
+import boto3
+import ipaddress
-
+ import json
++import sys
+ from botocore import exceptions
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
-import boto3
-import ipaddress
-
parser = argparse.ArgumentParser()
parser.add_argument('--vpc_id', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_download_git_certfile.py
index 8051e6d,ae9f28d..2e5efeb
--- a/infrastructure-provisioning/src/general/scripts/aws/common_download_git_certfile.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_download_git_certfile.py
@@@ -22,10 -22,10 +22,9 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
- from fabric.api import *
+import os
-
+ from datalab.actions_lib import *
+ from fabric.api import *
-import os
-
parser = argparse.ArgumentParser()
parser.add_argument('--keyfile', type=str, default='')
@@@ -33,7 -33,7 +32,6 @@@ parser.add_argument('--notebook_ip', ty
parser.add_argument('--os_user', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
create_aws_config_files()
env.hosts = "{}".format(args.notebook_ip)
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
index 1d0df4f,decc617..ec2e42d
--- a/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine-service.py
@@@ -21,24 -21,24 +21,23 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import traceback
+import json
++import logging
+ import os
-import uuid
+import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
+import traceback
- import os
- import uuid
from fabric.api import *
def clear_resources():
- emr_id = dlab.meta_lib.get_emr_id_by_name(notebook_config['cluster_name'])
- dlab.actions_lib.terminate_emr(emr_id)
- dlab.actions_lib. remove_kernels(notebook_config['cluster_name'], notebook_config['tag_name'],
- os.environ['notebook_instance_name'], os.environ['conf_os_user'],
- notebook_config['key_path'], os.environ['emr_version'])
+ emr_id = datalab.meta_lib.get_emr_id_by_name(notebook_config['cluster_name'])
+ datalab.actions_lib.terminate_emr(emr_id)
- datalab.actions_lib. remove_kernels(notebook_config['cluster_name'], notebook_config['tag_name'],
- os.environ['notebook_instance_name'], os.environ['conf_os_user'],
- notebook_config['key_path'], os.environ['emr_version'])
++ datalab.actions_lib.remove_kernels(notebook_config['cluster_name'], notebook_config['tag_name'],
++ os.environ['notebook_instance_name'], os.environ['conf_os_user'],
++ notebook_config['key_path'], os.environ['emr_version'])
if __name__ == "__main__":
@@@ -50,28 -50,28 +49,29 @@@
filename=local_log_filepath)
try:
# generating variables dictionary
- dlab.actions_lib.create_aws_config_files()
+ datalab.actions_lib.create_aws_config_files()
print('Generating infrastructure names and tags')
notebook_config = dict()
- notebook_config['service_base_name'] = os.environ['conf_service_base_name'] = dlab.fab.replace_multi_symbols(
- os.environ['conf_service_base_name'][:20], '-', True)
+ notebook_config['service_base_name'] = os.environ['conf_service_base_name'] = datalab.fab.replace_multi_symbols(
- os.environ['conf_service_base_name'][:20], '-', True)
++ os.environ['conf_service_base_name'][:20], '-', True)
notebook_config['notebook_name'] = os.environ['notebook_instance_name']
notebook_config['tag_name'] = notebook_config['service_base_name'] + '-tag'
notebook_config['project_name'] = os.environ['project_name']
notebook_config['endpoint_name'] = os.environ['endpoint_name']
notebook_config['bucket_name'] = '{0}-{1}-{2}-bucket'.format(notebook_config['service_base_name'],
-- notebook_config['project_name'],
-- notebook_config['endpoint_name']
++ notebook_config['project_name'],
++ notebook_config['endpoint_name']
).lower().replace('_', '-')
- notebook_config['cluster_name'] = dlab.meta_lib.get_not_configured_emr(notebook_config['tag_name'],
- notebook_config['notebook_name'], True)
- notebook_config['notebook_ip'] = dlab.meta_lib.get_instance_ip_address(
+ notebook_config['cluster_name'] = datalab.meta_lib.get_not_configured_emr(notebook_config['tag_name'],
- notebook_config['notebook_name'], True)
++ notebook_config['notebook_name'],
++ True)
+ notebook_config['notebook_ip'] = datalab.meta_lib.get_instance_ip_address(
notebook_config['tag_name'], notebook_config['notebook_name']).get('Private')
notebook_config['key_path'] = os.environ['conf_key_dir'] + '/' + os.environ['conf_key_name'] + '.pem'
- notebook_config['cluster_id'] = dlab.meta_lib.get_emr_id_by_name(notebook_config['cluster_name'])
+ notebook_config['cluster_id'] = datalab.meta_lib.get_emr_id_by_name(notebook_config['cluster_name'])
edge_instance_name = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
os.environ['project_name'], os.environ['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
if os.environ['application'] == 'deeplearning':
application = 'jupyter'
else:
@@@ -94,9 -94,9 +94,9 @@@
os.environ['application'], os.environ['conf_pypi_mirror'])
try:
local("~/scripts/{}_{}.py {}".format(application, 'install_dataengine-service_kernels', params))
- dlab.actions_lib.remove_emr_tag(notebook_config['cluster_id'], ['State'])
- dlab.actions_lib.tag_emr_volume(notebook_config['cluster_id'], notebook_config['cluster_name'],
- os.environ['conf_tag_resource_id'])
+ datalab.actions_lib.remove_emr_tag(notebook_config['cluster_id'], ['State'])
+ datalab.actions_lib.tag_emr_volume(notebook_config['cluster_id'], notebook_config['cluster_name'],
- os.environ['conf_tag_resource_id'])
++ os.environ['conf_tag_resource_id'])
except:
traceback.print_exc()
raise Exception
@@@ -116,9 -116,9 +116,9 @@@
os.environ['conf_os_user'])
try:
local("~/scripts/{0}.py {1}".format('common_configure_spark', params))
- dlab.actions_lib.remove_emr_tag(notebook_config['cluster_id'], ['State'])
- dlab.actions_lib.tag_emr_volume(notebook_config['cluster_id'], notebook_config['cluster_name'],
- os.environ['conf_tag_resource_id'])
+ datalab.actions_lib.remove_emr_tag(notebook_config['cluster_id'], ['State'])
+ datalab.actions_lib.tag_emr_volume(notebook_config['cluster_id'], notebook_config['cluster_name'],
- os.environ['conf_tag_resource_id'])
++ os.environ['conf_tag_resource_id'])
except:
traceback.print_exc()
raise Exception
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine.py
index c80328b,7ea51e3..c1f53b9
--- a/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_notebook_configure_dataengine.py
@@@ -21,15 -21,15 +21,14 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import traceback
+import json
++import logging
+ import os
-import uuid
+import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
+import traceback
- import os
- import uuid
from fabric.api import *
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
index 5a1a6f0,e817ff1..9724d56
--- a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
@@@ -21,15 -21,15 +21,13 @@@
#
# ******************************************************************************
--import logging
++import argparse
++import datalab.meta_lib
import json
--import sys
++import logging
import os
--import argparse
++import sys
import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
from fabric.api import *
parser = argparse.ArgumentParser()
@@@ -54,17 -54,17 +52,18 @@@ if __name__ == "__main__"
notebook_config['edge_name'] = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'],
notebook_config['endpoint_name'])
- edge_status = dlab.meta_lib.get_instance_status(notebook_config['service_base_name'] + '-tag',
- notebook_config['edge_name'])
+ edge_status = datalab.meta_lib.get_instance_status(notebook_config['service_base_name'] + '-tag',
- notebook_config['edge_name'])
++ notebook_config['edge_name'])
if edge_status != 'running':
logging.info('ERROR: Edge node is unavailable! Aborting...')
print('ERROR: Edge node is unavailable! Aborting...')
- notebook_config['ssn_hostname'] = dlab.meta_lib.get_instance_hostname(
+ notebook_config['ssn_hostname'] = datalab.meta_lib.get_instance_hostname(
'{}-tag'.format(notebook_config['service_base_name']),
'{}-ssn'.format(notebook_config['service_base_name']))
- dlab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_dlab_path'], os.environ['conf_os_user'],
- datalab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_datalab_path'], os.environ['conf_os_user'],
-- notebook_config['ssn_hostname'])
- dlab.fab.append_result("Edge node is unavailable")
++ datalab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_datalab_path'],
++ os.environ['conf_os_user'],
++ notebook_config['ssn_hostname'])
+ datalab.fab.append_result("Edge node is unavailable")
sys.exit(1)
print('Generating infrastructure names and tags')
try:
@@@ -138,13 -138,13 +137,13 @@@
params = "--node_name {} --ami_id {} --instance_type {} --key_name {} --security_group_ids {} --subnet_id {} " \
"--iam_profile {} --infra_tag_name {} --infra_tag_value {} --instance_class {} " \
"--instance_disk_size {} --primary_disk_size {}" .format(
-- notebook_config['instance_name'], notebook_config['ami_id'], notebook_config['instance_type'],
-- notebook_config['key_name'],
- dlab.meta_lib.get_security_group_by_name(notebook_config['security_group_name']),
- dlab.meta_lib.get_subnet_by_cidr(notebook_config['subnet_cidr'], os.environ['aws_notebook_vpc_id']),
- datalab.meta_lib.get_security_group_by_name(notebook_config['security_group_name']),
- datalab.meta_lib.get_subnet_by_cidr(notebook_config['subnet_cidr'], os.environ['aws_notebook_vpc_id']),
-- notebook_config['role_profile_name'],
-- notebook_config['tag_name'], notebook_config['instance_name'], instance_class,
-- os.environ['notebook_disk_size'], notebook_config['primary_disk_size'])
++ notebook_config['instance_name'], notebook_config['ami_id'], notebook_config['instance_type'],
++ notebook_config['key_name'],
++ datalab.meta_lib.get_security_group_by_name(notebook_config['security_group_name']),
++ datalab.meta_lib.get_subnet_by_cidr(notebook_config['subnet_cidr'], os.environ['aws_notebook_vpc_id']),
++ notebook_config['role_profile_name'],
++ notebook_config['tag_name'], notebook_config['instance_name'], instance_class,
++ os.environ['notebook_disk_size'], notebook_config['primary_disk_size'])
try:
local("~/scripts/{}.py {}".format('common_create_instance', params))
@@@ -152,6 -152,6 +151,5 @@@
traceback.print_exc()
raise Exception
except Exception as err:
- dlab.fab.append_result("Failed to create instance.", str(err))
+ datalab.fab.append_result("Failed to create instance.", str(err))
sys.exit(1)
--
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_put_to_bucket.py
index 8f433fe,8a94e4d..9ccb627
--- a/infrastructure-provisioning/src/general/scripts/aws/common_put_to_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_put_to_bucket.py
@@@ -22,8 -22,8 +22,8 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
-from datalab.actions_lib import *
import sys
++from datalab.actions_lib import *
parser = argparse.ArgumentParser()
parser.add_argument('--bucket_name', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_remove_remote_kernels.py
index 826bff4,a1f91f0..00b7900
--- a/infrastructure-provisioning/src/general/scripts/aws/common_remove_remote_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_remove_remote_kernels.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
--import sys
import argparse
-from fabric.api import *
-from datalab.fab import find_cluster_kernels
++import sys
+ from datalab.actions_lib import *
++from datalab.fab import find_cluster_kernels
+from fabric.api import *
- from dlab.fab import find_cluster_kernels
- from dlab.actions_lib import *
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_reupload_key.py
index 3fd8adf,16f0039..24a4d6d
--- a/infrastructure-provisioning/src/general/scripts/aws/common_reupload_key.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_reupload_key.py
@@@ -23,11 -23,11 +23,10 @@@
import argparse
-from fabric.api import *
+ from datalab.actions_lib import *
-from datalab.meta_lib import *
+ from datalab.fab import *
-import json
++from datalab.meta_lib import *
+from fabric.api import *
- from dlab.actions_lib import *
- from dlab.meta_lib import *
- from dlab.fab import *
- import json
parser = argparse.ArgumentParser()
parser.add_argument('--conf_resource', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
index d153082,a194955..aa97171
--- a/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_start_notebook.py
@@@ -21,19 -21,19 +21,14 @@@
#
# ******************************************************************************
--import logging
++import datalab.meta_lib
import json
++import logging
++import os
import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
import traceback
--import os
--import uuid
--import argparse
from fabric.api import *
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id'])
@@@ -95,9 -95,9 +90,9 @@@
sys.exit(1)
try:
- ip_address = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['notebook_name']).get('Private')
- dns_name = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['notebook_name'])
+ ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['notebook_name']).get('Private')
++ notebook_config['notebook_name']).get('Private')
+ dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['notebook_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['notebook_name']))
@@@ -112,7 -112,7 +107,5 @@@
print(json.dumps(res))
result.write(json.dumps(res))
except Exception as err:
- dlab.fab.append_result("Error with writing results", str(err))
+ datalab.fab.append_result("Error with writing results", str(err))
sys.exit(1)
--
--
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_stop_notebook.py
index 679d4eb,cb6b213..f890f40
--- a/infrastructure-provisioning/src/general/scripts/aws/common_stop_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_stop_notebook.py
@@@ -21,19 -21,19 +21,15 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
++import boto3
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-from fabric.api import *
-import traceback
+import json
- import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- from fabric.api import *
- import traceback
++import logging
import os
--import uuid
--import boto3
--import argparse
import sys
++from fabric.api import *
def stop_notebook(nb_tag_value, bucket_name, tag_name, ssh_user, key_path):
@@@ -51,12 -51,12 +47,12 @@@
for tag in cluster.get('Tags'):
if tag.get('Key') == 'ComputationalName':
computational_name = tag.get('Value')
- dlab.actions_lib.s3_cleanup(bucket_name, emr_name, os.environ['project_name'])
+ datalab.actions_lib.s3_cleanup(bucket_name, emr_name, os.environ['project_name'])
print("The bucket {} has been cleaned successfully".format(bucket_name))
- dlab.actions_lib.terminate_emr(cluster_id)
+ datalab.actions_lib.terminate_emr(cluster_id)
print("The EMR cluster {} has been terminated successfully".format(emr_name))
- dlab.actions_lib.remove_kernels(emr_name, tag_name, nb_tag_value, ssh_user, key_path, emr_version,
- computational_name)
+ datalab.actions_lib.remove_kernels(emr_name, tag_name, nb_tag_value, ssh_user, key_path, emr_version,
- computational_name)
++ computational_name)
print("{} kernels have been removed from notebook successfully".format(emr_name))
else:
print("There are no EMR clusters to terminate.")
@@@ -130,6 -130,6 +126,5 @@@ if __name__ == "__main__"
print(json.dumps(res))
result.write(json.dumps(res))
except Exception as err:
- dlab.fab.append_result("Error with writing results", str(err))
+ datalab.fab.append_result("Error with writing results", str(err))
sys.exit(1)
--
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
index c199089,4a7be6e..ca0af77
--- a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
@@@ -21,16 -21,16 +21,15 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
++import boto3
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import traceback
+import json
++import logging
+ import os
-import boto3
-import uuid
+import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
+import traceback
- import os
- import boto3
- import uuid
def terminate_nb(nb_tag_value, bucket_name, tag_name):
diff --cc infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook_image.py
index 3da4f63,3a1a1b6..cefc22e
--- a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook_image.py
@@@ -21,13 -21,13 +21,10 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import sys
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import sys
import json
import os
--
++import sys
if __name__ == "__main__":
try:
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
index 147247a,1088e41..eed5b4a
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
@@@ -21,20 -21,20 +21,18 @@@
#
# ******************************************************************************
-import json
-import time
-from fabric.api import *
-import datalab.fab
++import argparse
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import traceback
-import sys
-import os
+import json
- import time
- from fabric.api import *
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import traceback
- import sys
- import os
import logging
--import argparse
import multiprocessing
- from dlab.common_lib import manage_pkg
-
++import os
++import sys
++import traceback
+ from datalab.common_lib import manage_pkg
-
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--uuid', type=str, default='')
@@@ -81,8 -81,8 +79,8 @@@ def configure_dataengine_service(instan
logging.info('[CONFIGURE DATAENGINE SERVICE]')
print('[CONFIGURE DATAENGINE SERVICE]')
try:
- dlab.fab.configure_data_engine_service_pip(emr_conf['instance_ip'], emr_conf['os_user'],
- emr_conf['key_path'], True)
+ datalab.fab.configure_data_engine_service_pip(emr_conf['instance_ip'], emr_conf['os_user'],
- emr_conf['key_path'], True)
++ emr_conf['key_path'], True)
env['connection_attempts'] = 100
env.key_filename = emr_conf['key_path']
env.host_string = emr_conf['os_user'] + '@' + emr_conf['instance_ip']
@@@ -217,16 -217,16 +215,16 @@@ if __name__ == "__main__"
emr_conf['project_name'],
emr_conf['endpoint_name'])
emr_conf['vpc_id'] = os.environ['aws_vpc_id']
- emr_conf['cluster_id'] = dlab.meta_lib.get_emr_id_by_name(emr_conf['cluster_name'])
- emr_conf['cluster_instances'] = dlab.meta_lib.get_emr_instances_list(emr_conf['cluster_id'])
- emr_conf['cluster_master_instances'] = dlab.meta_lib.get_emr_instances_list(emr_conf['cluster_id'], 'MASTER')
- emr_conf['cluster_core_instances'] = dlab.meta_lib.get_emr_instances_list(emr_conf['cluster_id'], 'CORE')
+ emr_conf['cluster_id'] = datalab.meta_lib.get_emr_id_by_name(emr_conf['cluster_name'])
+ emr_conf['cluster_instances'] = datalab.meta_lib.get_emr_instances_list(emr_conf['cluster_id'])
+ emr_conf['cluster_master_instances'] = datalab.meta_lib.get_emr_instances_list(emr_conf['cluster_id'], 'MASTER')
+ emr_conf['cluster_core_instances'] = datalab.meta_lib.get_emr_instances_list(emr_conf['cluster_id'], 'CORE')
emr_conf['edge_instance_name'] = '{0}-{1}-{2}-edge'.format(emr_conf['service_base_name'],
emr_conf['project_name'], emr_conf['endpoint_name'])
- emr_conf['edge_instance_hostname'] = dlab.meta_lib.get_instance_private_ip_address(
+ emr_conf['edge_instance_hostname'] = datalab.meta_lib.get_instance_private_ip_address(
emr_conf['tag_name'], emr_conf['edge_instance_name'])
- emr_conf['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(emr_conf['tag_name'],
- emr_conf['edge_instance_name'])
+ emr_conf['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(emr_conf['tag_name'],
- emr_conf['edge_instance_name'])
++ emr_conf['edge_instance_name'])
emr_conf['user_keyname'] = emr_conf['project_name']
emr_conf['os_user'] = os.environ['conf_os_user']
emr_conf['initial_user'] = 'ec2-user'
@@@ -280,8 -280,8 +278,8 @@@
"computational_url": [
{"description": "EMR Master",
"url": emr_master_acces_url},
-- #{"description": "EMR Master (via tunnl)",
-- #"url": emr_master_url}
++ # {"description": "EMR Master (via tunnl)",
++ # "url": emr_master_url}
]}
print(json.dumps(res))
result.write(json.dumps(res))
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
index 0cdba1f,df9d989..86c430c
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_create.py
@@@ -21,19 -21,19 +21,18 @@@
#
# ******************************************************************************
--import boto3
--from botocore.client import Config
import argparse
++import ast
++import boto3
++import logging
import re
--import time
import sys
--from fabric.api import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
-from datalab.meta_lib import *
-from datalab.actions_lib import *
--import json
++import time
import traceback
--import logging
--import ast
++from botocore.client import Config
++from datalab.actions_lib import *
++from datalab.meta_lib import *
++from fabric.api import *
parser = argparse.ArgumentParser()
# parser.add_argument('--id', type=str, default='')
@@@ -41,8 -41,8 +40,10 @@@ parser.add_argument('--dry_run', action
parser.add_argument('--name', type=str, default='', help='Name to be applied to Cluster ( MANDATORY !!! )')
parser.add_argument('--applications', type=str, default='',
help='Set of applications to be installed on EMR (Default are: "Hadoop Hive Hue Spark Livy")')
--parser.add_argument('--master_instance_type', type=str, default='', help='EC2 instance size for Master-Node (Default: m3.xlarge)')
--parser.add_argument('--slave_instance_type', type=str, default='', help='EC2 instance size for Worker-Nodes (Default: m3.xlarge)')
++parser.add_argument('--master_instance_type', type=str, default='',
++ help='EC2 instance size for Master-Node (Default: m3.xlarge)')
++parser.add_argument('--slave_instance_type', type=str, default='',
++ help='EC2 instance size for Worker-Nodes (Default: m3.xlarge)')
parser.add_argument('--instance_count', type=int, default='',
help='Number of nodes the cluster will consist of (Default: 3)')
parser.add_argument('--release_label', type=str, default='', help='EMR release version (Default: "emr-4.8.0")')
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
index a8a1e47,a69b140..04a9107
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_install_libs.py
@@@ -21,20 -21,20 +21,20 @@@
#
# ******************************************************************************
++import logging
++import multiprocessing
import os
import sys
--import logging
import traceback
- from dlab.fab import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
++from datalab.actions_lib import *
+ from datalab.fab import *
+ from datalab.meta_lib import *
-from datalab.actions_lib import *
from fabric.api import *
--import multiprocessing
def install_libs(instance, data_engine):
data_engine['instance_ip'] = instance.get('PrivateIpAddress')
-- params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}"'\
++ params = '--os_user {} --instance_ip {} --keyfile "{}" --libs "{}"' \
.format(data_engine['os_user'], data_engine['instance_ip'],
data_engine['keyfile'], data_engine['libs'])
try:
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
index 9360375,a9c5364..0a80431
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_list_libs.py
@@@ -21,13 -21,13 +21,13 @@@
#
# ******************************************************************************
++import logging
import os
import sys
--import logging
import traceback
- from dlab.fab import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
++from datalab.actions_lib import *
+ from datalab.fab import *
+ from datalab.meta_lib import *
-from datalab.actions_lib import *
from fabric.api import *
if __name__ == "__main__":
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
index 3a08ba4,ecb3ed1..faf97c8
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
@@@ -21,24 -21,24 +21,20 @@@
#
# ******************************************************************************
++import argparse
++import datalab.meta_lib
import json
++import logging
++import os
++import sys
import time
--from fabric.api import *
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
import traceback
--import argparse
--import sys
--import os
--import logging
--
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--uuid', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id'])
@@@ -65,13 -65,13 +61,13 @@@
if edge_status != 'running':
logging.info('ERROR: Edge node is unavailable! Aborting...')
print('ERROR: Edge node is unavailable! Aborting...')
- ssn_hostname = dlab.meta_lib.get_instance_hostname(
+ ssn_hostname = datalab.meta_lib.get_instance_hostname(
emr_conf['service_base_name'] + '-tag',
emr_conf['service_base_name'] + '-ssn')
- dlab.fab.put_resource_status('edge', 'Unavailable',
- os.environ['ssn_dlab_path'],
- os.environ['conf_os_user'], ssn_hostname)
- dlab.fab.append_result("Edge node is unavailable")
+ datalab.fab.put_resource_status('edge', 'Unavailable',
- os.environ['ssn_datalab_path'],
- os.environ['conf_os_user'], ssn_hostname)
++ os.environ['ssn_datalab_path'],
++ os.environ['conf_os_user'], ssn_hostname)
+ datalab.fab.append_result("Edge node is unavailable")
sys.exit(1)
print('Generating infrastructure names and tags')
if 'computational_name' in os.environ:
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine-service_terminate.py
index e9551e3,d6b86f5..0820e39
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_terminate.py
@@@ -21,16 -21,16 +21,15 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
+import boto3
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import boto3
++import json
import logging
--import argparse
--import sys
import os
++import sys
import traceback
--import json
def terminate_emr_cluster(emr_name, bucket_name, tag_name, nb_tag_value, ssh_user, key_path):
@@@ -48,13 -48,13 +47,13 @@@
for tag in cluster.get('Tags'):
if tag.get('Key') == 'ComputationalName':
computational_name = tag.get('Value')
- dlab.actions_lib.s3_cleanup(bucket_name, emr_name, os.environ['project_name'])
+ datalab.actions_lib.s3_cleanup(bucket_name, emr_name, os.environ['project_name'])
print("The bucket {} has been cleaned successfully".format(bucket_name))
- dlab.actions_lib.terminate_emr(cluster_id)
+ datalab.actions_lib.terminate_emr(cluster_id)
print("The EMR cluster {} has been terminated successfully".format(emr_name))
print("Removing EMR kernels from notebook")
- dlab.actions_lib.remove_kernels(emr_name, tag_name, nb_tag_value, ssh_user, key_path,
- emr_version, computational_name)
+ datalab.actions_lib.remove_kernels(emr_name, tag_name, nb_tag_value, ssh_user, key_path,
- emr_version, computational_name)
++ emr_version, computational_name)
else:
print("There are no EMR clusters to terminate.")
except:
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
index e0a4f0c,0aecbc2..d34c94e
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
@@@ -21,19 -21,19 +21,16 @@@
#
# ******************************************************************************
-import json
-import time
-from fabric.api import *
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import traceback
-import sys
-import os
-import uuid
+import json
- import time
- from fabric.api import *
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import traceback
- import sys
- import os
- import uuid
import logging
--from Crypto.PublicKey import RSA
import multiprocessing
++import os
++import sys
++import traceback
++from fabric.api import *
def configure_slave(slave_number, data_engine):
@@@ -193,18 -193,18 +190,18 @@@ if __name__ == "__main__"
data_engine['notebook_dataengine_role_profile_name'] = '{}-{}-{}-nb-de-profile' \
.format(data_engine['service_base_name'], data_engine['project_name'], data_engine['endpoint_name'])
data_engine['instance_count'] = int(os.environ['dataengine_instance_count'])
- master_node_hostname = dlab.meta_lib.get_instance_hostname(data_engine['tag_name'],
- data_engine['master_node_name'])
- data_engine['dlab_ssh_user'] = os.environ['conf_os_user']
+ master_node_hostname = datalab.meta_lib.get_instance_hostname(data_engine['tag_name'],
- data_engine['master_node_name'])
++ data_engine['master_node_name'])
+ data_engine['datalab_ssh_user'] = os.environ['conf_os_user']
data_engine['user_keyname'] = data_engine['project_name']
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
edge_instance_name = '{0}-{1}-{2}-edge'.format(data_engine['service_base_name'],
data_engine['project_name'], data_engine['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(data_engine['tag_name'], edge_instance_name)
- edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(data_engine['tag_name'],
- edge_instance_name).get('Private')
- data_engine['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(data_engine['tag_name'],
- edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(data_engine['tag_name'], edge_instance_name)
+ edge_instance_private_ip = datalab.meta_lib.get_instance_ip_address(data_engine['tag_name'],
- edge_instance_name).get('Private')
++ edge_instance_name).get('Private')
+ data_engine['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(data_engine['tag_name'],
- edge_instance_name)
++ edge_instance_name)
if os.environ['conf_os_family'] == 'debian':
data_engine['initial_user'] = 'ubuntu'
data_engine['sudo_group'] = 'sudo'
@@@ -275,8 -275,8 +272,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES ON MASTER NODE]')
print('[INSTALLING PREREQUISITES ON MASTER NODE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(master_node_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(master_node_hostname, keyfile_name, data_engine['datalab_ssh_user'], data_engine['region'],
edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -342,8 -342,8 +339,8 @@@
try:
print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
- notebook_instance_ip = dlab.meta_lib.get_instance_private_ip_address('Name',
- os.environ['notebook_instance_name'])
+ notebook_instance_ip = datalab.meta_lib.get_instance_private_ip_address('Name',
- os.environ['notebook_instance_name'])
++ os.environ['notebook_instance_name'])
additional_info = {
"computational_name": data_engine['computational_name'],
"master_node_hostname": master_node_hostname,
@@@ -376,8 -376,8 +373,8 @@@
sys.exit(1)
try:
- ip_address = dlab.meta_lib.get_instance_ip_address(data_engine['tag_name'],
- data_engine['master_node_name']).get('Private')
+ ip_address = datalab.meta_lib.get_instance_ip_address(data_engine['tag_name'],
- data_engine['master_node_name']).get('Private')
++ data_engine['master_node_name']).get('Private')
spark_master_url = "http://" + ip_address + ":8080"
spark_master_access_url = "https://{}/{}_{}/".format(data_engine['edge_instance_hostname'],
data_engine['exploratory_name'],
@@@ -392,8 -392,8 +389,8 @@@
print("Instance count: {}".format(str(data_engine['instance_count'])))
with open("/root/result.json", 'w') as result:
res = {"hostname": data_engine['cluster_name'],
- "instance_id": dlab.meta_lib.get_instance_by_name(data_engine['tag_name'],
- data_engine['master_node_name']),
+ "instance_id": datalab.meta_lib.get_instance_by_name(data_engine['tag_name'],
- data_engine['master_node_name']),
++ data_engine['master_node_name']),
"key_name": data_engine['key_name'],
"Action": "Create new Data Engine",
"computational_url": [
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
index ad19f7a,b9d5cb8..27e2eef
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_prepare.py
@@@ -21,20 -21,20 +21,13 @@@
#
# ******************************************************************************
-import json
-import time
-from fabric.api import *
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import traceback
-import sys
-import os
-import uuid
+import json
- import time
- from fabric.api import *
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import traceback
- import sys
- import os
- import uuid
import logging
--from Crypto.PublicKey import RSA
--import multiprocessing
--
++import os
++import sys
++import traceback
++from fabric.api import *
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -55,11 -55,11 +48,12 @@@
if edge_status != 'running':
logging.info('ERROR: Edge node is unavailable! Aborting...')
print('ERROR: Edge node is unavailable! Aborting...')
- ssn_hostname = dlab.meta_lib.get_instance_hostname(data_engine['service_base_name'] + '-tag',
- data_engine['service_base_name'] + '-ssn')
- dlab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_dlab_path'], os.environ['conf_os_user'],
- ssn_hostname)
- dlab.fab.append_result("Edge node is unavailable")
+ ssn_hostname = datalab.meta_lib.get_instance_hostname(data_engine['service_base_name'] + '-tag',
- data_engine['service_base_name'] + '-ssn')
- datalab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_datalab_path'], os.environ['conf_os_user'],
- ssn_hostname)
++ data_engine['service_base_name'] + '-ssn')
++ datalab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_datalab_path'],
++ os.environ['conf_os_user'],
++ ssn_hostname)
+ datalab.fab.append_result("Edge node is unavailable")
sys.exit(1)
print('Generating infrastructure names and tags')
if 'exploratory_name' in os.environ:
@@@ -161,12 -161,12 +155,12 @@@
data_engine['master_node_name'], data_engine['primary_disk_size'], data_engine['instance_class'])
try:
local("~/scripts/{}.py {}".format('common_create_instance', params))
- data_engine['master_id'] = dlab.meta_lib.get_instance_by_name(data_engine['tag_name'],
- data_engine['master_node_name'])
- dlab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_tag'], False)
- dlab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_resource_tag'], False)
- dlab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_billing_tag'], False)
- dlab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_tag_type'], False)
+ data_engine['master_id'] = datalab.meta_lib.get_instance_by_name(data_engine['tag_name'],
- data_engine['master_node_name'])
++ data_engine['master_node_name'])
+ datalab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_tag'], False)
+ datalab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_resource_tag'], False)
+ datalab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_billing_tag'], False)
+ datalab.actions_lib.create_tag(data_engine['master_id'], data_engine['cluster_nodes_tag_type'], False)
except:
traceback.print_exc()
raise Exception
@@@ -185,17 -185,17 +179,20 @@@
"--primary_disk_size {} --instance_class {}" \
.format(slave_name, data_engine['ami_id'], data_engine['slave_size'],
data_engine['key_name'],
- dlab.meta_lib.get_security_group_by_name(data_engine['dataengine_slave_security_group_name']),
- dlab.meta_lib.get_subnet_by_cidr(data_engine['subnet_cidr'], os.environ['aws_notebook_vpc_id']),
- datalab.meta_lib.get_security_group_by_name(data_engine['dataengine_slave_security_group_name']),
- datalab.meta_lib.get_subnet_by_cidr(data_engine['subnet_cidr'], os.environ['aws_notebook_vpc_id']),
++ datalab.meta_lib.get_security_group_by_name(
++ data_engine['dataengine_slave_security_group_name']),
++ datalab.meta_lib.get_subnet_by_cidr(data_engine['subnet_cidr'],
++ os.environ['aws_notebook_vpc_id']),
data_engine['notebook_dataengine_role_profile_name'], data_engine['tag_name'],
slave_name, data_engine['primary_disk_size'], data_engine['instance_class'])
try:
local("~/scripts/{}.py {}".format('common_create_instance', params))
- data_engine['slave_id'] = dlab.meta_lib.get_instance_by_name(data_engine['tag_name'], slave_name)
- dlab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_tag'], False)
- dlab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_resource_tag'], False)
- dlab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_billing_tag'], False)
- dlab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_tag_type'], False)
+ data_engine['slave_id'] = datalab.meta_lib.get_instance_by_name(data_engine['tag_name'], slave_name)
+ datalab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_tag'], False)
- datalab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_resource_tag'], False)
++ datalab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_resource_tag'],
++ False)
+ datalab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_billing_tag'], False)
+ datalab.actions_lib.create_tag(data_engine['slave_id'], data_engine['cluster_nodes_tag_type'], False)
except:
traceback.print_exc()
raise Exception
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
index 0450ff7,e3150b8..60b0e3e
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_start.py
@@@ -21,12 -21,12 +21,12 @@@
#
# ******************************************************************************
--import logging
-import json
-import os
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+import json
++import logging
+import os
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
import sys
import traceback
from fabric.api import *
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine_stop.py
index d31d395,f0f60f4..1d540d9
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_stop.py
@@@ -21,11 -21,11 +21,11 @@@
#
# ******************************************************************************
--import logging
++import datalab.actions_lib
++import datalab.fab
import json
++import logging
import os
- import dlab.fab
- import dlab.actions_lib
-import datalab.fab
-import datalab.actions_lib
import sys
diff --cc infrastructure-provisioning/src/general/scripts/aws/dataengine_terminate.py
index 7d8c10d,dc3a4d2..84e9b3d
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_terminate.py
@@@ -21,13 -21,13 +21,13 @@@
#
# ******************************************************************************
--import logging
++import datalab.actions_lib
++import datalab.fab
import json
++import logging
++import os
import sys
- import dlab.fab
- import dlab.actions_lib
-import datalab.fab
-import datalab.actions_lib
import traceback
--import os
def terminate_data_engine(tag_name, notebook_name,
@@@ -41,8 -41,8 +41,8 @@@
print("Removing Data Engine kernels from notebook")
try:
- dlab.actions_lib.remove_dataengine_kernels(tag_name, notebook_name,
- os_user, key_path, remote_kernel_name)
+ datalab.actions_lib.remove_dataengine_kernels(tag_name, notebook_name,
- os_user, key_path, remote_kernel_name)
++ os_user, key_path, remote_kernel_name)
except:
sys.exit(1)
diff --cc infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
index a2ca856,cfd6863..e3fca38
--- a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
@@@ -21,15 -21,15 +21,13 @@@
#
# ******************************************************************************
--import logging
++import argparse
++import datalab.meta_lib
import json
++import logging
++import os
import sys
import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
--import os
--import argparse
from fabric.api import *
parser = argparse.ArgumentParser()
@@@ -86,20 -86,20 +84,21 @@@ if __name__ == "__main__"
notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
# generating variables regarding EDGE proxy on Notebook instance
- instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
+ instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
++ notebook_config['instance_name'])
edge_instance_name = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'], notebook_config['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
- edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
- notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
++ edge_instance_name).get('Private')
+ notebook_config['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
++ edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
- edge_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
- edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
++ edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get(
++ 'Private')
except Exception as err:
- dlab.fab.append_result("Failed to generate variables dictionary.", str(err))
- dlab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
+ datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
try:
@@@ -186,11 -186,11 +185,11 @@@
"--scala_version {4} --spark_version {5} " \
"--hadoop_version {6} --region {7} " \
"--r_mirror {8} --ip_address {9} --exploratory_name {10} --edge_ip {11}" \
- .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'],
- .format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'],
-- os.environ['notebook_jupyter_version'], os.environ['notebook_scala_version'],
-- os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'],
-- os.environ['aws_region'], os.environ['notebook_r_mirror'],
-- notebook_config['ip_address'], notebook_config['exploratory_name'], edge_ip)
++ .format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'],
++ os.environ['notebook_jupyter_version'], os.environ['notebook_scala_version'],
++ os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'],
++ os.environ['aws_region'], os.environ['notebook_r_mirror'],
++ notebook_config['ip_address'], notebook_config['exploratory_name'], edge_ip)
try:
local("~/scripts/{}.py {}".format('configure_deep_learning_node', params))
except:
@@@ -299,9 -299,9 +298,9 @@@
sys.exit(1)
try:
# generating output information
- ip_address = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
- dns_name = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
++ notebook_config['instance_name']).get('Private')
+ dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
tensor_board_url = 'http://' + ip_address + ':6006'
jupyter_url = 'http://' + ip_address + ':8888/{}/'.format(notebook_config['exploratory_name'])
jupyter_notebook_access_url = "https://{}/{}/".format(notebook_config['edge_instance_hostname'],
@@@ -316,8 -316,8 +315,8 @@@
print("Instance name: {}".format(notebook_config['instance_name']))
print("Private DNS: {}".format(dns_name))
print("Private IP: {}".format(ip_address))
- print("Instance ID: {}".format(dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
+ print("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
++ notebook_config['instance_name'])))
print("Instance type: {}".format(notebook_config['instance_type']))
print("Key name: {}".format(notebook_config['key_name']))
print("User key name: {}".format(notebook_config['user_keyname']))
@@@ -327,15 -327,15 +326,15 @@@
print("Ungit URL: {}".format(ungit_ip_url))
print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.
- format(notebook_config['key_name'],notebook_config['dlab_ssh_user'], ip_address))
- format(notebook_config['key_name'],notebook_config['datalab_ssh_user'], ip_address))
++ format(notebook_config['key_name'], notebook_config['datalab_ssh_user'], ip_address))
print('SSH access (from Edge node, via FQDN): ssh -i {0}.pem {1}@{2}'.
- format(notebook_config['key_name'], notebook_config['dlab_ssh_user'], dns_name))
+ format(notebook_config['key_name'], notebook_config['datalab_ssh_user'], dns_name))
with open("/root/result.json", 'w') as result:
res = {"hostname": dns_name,
"ip": ip_address,
- "instance_id": dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
+ "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
++ notebook_config['instance_name']),
"master_keyname": os.environ['conf_key_name'],
"notebook_name": notebook_config['instance_name'],
"notebook_image_name": notebook_config['notebook_image_name'],
diff --cc infrastructure-provisioning/src/general/scripts/aws/edge_associate_elastic_ip.py
index 131ac46,2eaba02..5363073
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_associate_elastic_ip.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_associate_elastic_ip.py
@@@ -21,12 -21,12 +21,12 @@@
#
# ******************************************************************************
- import sys
- from dlab.fab import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
- import os
+import argparse
++import os
+ import sys
++from datalab.actions_lib import *
+ from datalab.fab import *
+ from datalab.meta_lib import *
-from datalab.actions_lib import *
-import os
-import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--elastic_ip', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
index bafd031,0fe7a4a..91c1dde
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
@@@ -21,19 -21,19 +21,17 @@@
#
# ******************************************************************************
-import json
-import sys
-import time
-import os
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+import json
- import sys
- import time
- import os
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
import logging
++import os
++import sys
import traceback
import uuid
from fabric.api import *
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id'])
@@@ -42,14 -42,14 +40,15 @@@
level=logging.DEBUG,
filename=local_log_filepath)
++
def clear_resources():
- dlab.actions_lib.remove_all_iam_resources('notebook', edge_conf['project_name'])
- dlab.actions_lib.remove_all_iam_resources('edge', edge_conf['project_name'])
- dlab.actions_lib.remove_ec2(edge_conf['tag_name'], edge_conf['instance_name'])
- dlab.actions_lib.remove_sgroups(edge_conf['dataengine_instances_name'])
- dlab.actions_lib.remove_sgroups(edge_conf['notebook_instance_name'])
- dlab.actions_lib.remove_sgroups(edge_conf['instance_name'])
- dlab.actions_lib.remove_s3('edge', edge_conf['project_name'])
+ datalab.actions_lib.remove_all_iam_resources('notebook', edge_conf['project_name'])
+ datalab.actions_lib.remove_all_iam_resources('edge', edge_conf['project_name'])
+ datalab.actions_lib.remove_ec2(edge_conf['tag_name'], edge_conf['instance_name'])
+ datalab.actions_lib.remove_sgroups(edge_conf['dataengine_instances_name'])
+ datalab.actions_lib.remove_sgroups(edge_conf['notebook_instance_name'])
+ datalab.actions_lib.remove_sgroups(edge_conf['instance_name'])
+ datalab.actions_lib.remove_s3('edge', edge_conf['project_name'])
try:
print('Generating infrastructure names and tags')
@@@ -87,21 -87,21 +86,22 @@@
tag = {"Key": edge_conf['tag_name'],
"Value": "{}-{}-{}-subnet".format(edge_conf['service_base_name'], edge_conf['project_name'],
edge_conf['endpoint_name'])}
- edge_conf['private_subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag)
- edge_conf['dlab_ssh_user'] = os.environ['conf_os_user']
+ edge_conf['private_subnet_cidr'] = datalab.meta_lib.get_subnet_by_tag(tag)
+ edge_conf['datalab_ssh_user'] = os.environ['conf_os_user']
edge_conf['network_type'] = os.environ['conf_network_type']
if edge_conf['network_type'] == 'public':
- edge_conf['edge_public_ip'] = dlab.meta_lib.get_instance_ip_address(edge_conf['tag_name'],
- edge_conf['instance_name']).get('Public')
- edge_conf['edge_private_ip'] = dlab.meta_lib.get_instance_ip_address(
+ edge_conf['edge_public_ip'] = datalab.meta_lib.get_instance_ip_address(edge_conf['tag_name'],
- edge_conf['instance_name']).get('Public')
++ edge_conf['instance_name']).get(
++ 'Public')
+ edge_conf['edge_private_ip'] = datalab.meta_lib.get_instance_ip_address(
edge_conf['tag_name'], edge_conf['instance_name']).get('Private')
elif edge_conf['network_type'] == 'private':
- edge_conf['edge_private_ip'] = dlab.meta_lib.get_instance_ip_address(
+ edge_conf['edge_private_ip'] = datalab.meta_lib.get_instance_ip_address(
edge_conf['tag_name'], edge_conf['instance_name']).get('Private')
edge_conf['edge_public_ip'] = edge_conf['edge_private_ip']
- edge_conf['vpc1_cidrs'] = dlab.meta_lib.get_vpc_cidr_by_id(os.environ['aws_vpc_id'])
+ edge_conf['vpc1_cidrs'] = datalab.meta_lib.get_vpc_cidr_by_id(os.environ['aws_vpc_id'])
try:
- edge_conf['vpc2_cidrs'] = dlab.meta_lib.get_vpc_cidr_by_id(os.environ['aws_notebook_vpc_id'])
+ edge_conf['vpc2_cidrs'] = datalab.meta_lib.get_vpc_cidr_by_id(os.environ['aws_notebook_vpc_id'])
edge_conf['vpc_cidrs'] = list(set(edge_conf['vpc1_cidrs'] + edge_conf['vpc2_cidrs']))
except KeyError:
edge_conf['vpc_cidrs'] = list(set(edge_conf['vpc1_cidrs']))
@@@ -110,8 -110,8 +110,8 @@@
for cidr in os.environ['conf_allowed_ip_cidr'].split(','):
edge_conf['allowed_ip_cidr'].append(cidr.replace(' ', ''))
- edge_conf['instance_hostname'] = dlab.meta_lib.get_instance_hostname(edge_conf['tag_name'],
- edge_conf['instance_name'])
+ edge_conf['instance_hostname'] = datalab.meta_lib.get_instance_hostname(edge_conf['tag_name'],
- edge_conf['instance_name'])
++ edge_conf['instance_name'])
edge_conf['keyfile_name'] = "{}{}.pem".format(os.environ['conf_key_dir'], edge_conf['key_name'])
if os.environ['conf_stepcerts_enabled'] == 'true':
@@@ -153,8 -153,8 +153,8 @@@
try:
print('[INSTALLING PREREQUISITES]')
logging.info('[INSTALLING PREREQUISITES]')
-- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'],
++ params = "--hostname {} --keyfile {} --user {} --region {}". \
+ format(edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['datalab_ssh_user'],
os.environ['aws_region'])
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -232,10 -232,10 +232,10 @@@
edge_conf['keycloak_client_secret'] = str(uuid.uuid4())
params = "--hostname {} --keyfile {} --user {} --keycloak_client_id {} --keycloak_client_secret {} " \
"--step_cert_sans '{}' ".format(
- edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'],
- edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['datalab_ssh_user'],
-- '{}-{}-{}'.format(edge_conf['service_base_name'], edge_conf['project_name'],
-- edge_conf['endpoint_name']),
-- edge_conf['keycloak_client_secret'], edge_conf['step_cert_sans'])
++ edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['datalab_ssh_user'],
++ '{}-{}-{}'.format(edge_conf['service_base_name'], edge_conf['project_name'],
++ edge_conf['endpoint_name']),
++ edge_conf['keycloak_client_secret'], edge_conf['step_cert_sans'])
try:
local("~/scripts/{}.py {}".format('configure_nginx_reverse_proxy', params))
except:
@@@ -289,8 -289,8 +289,8 @@@
print("Hostname: {}".format(edge_conf['instance_hostname']))
print("Public IP: {}".format(edge_conf['edge_public_ip']))
print("Private IP: {}".format(edge_conf['edge_private_ip']))
- print("Instance ID: {}".format(dlab.meta_lib.get_instance_by_name(edge_conf['tag_name'],
- edge_conf['instance_name'])))
+ print("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(edge_conf['tag_name'],
- edge_conf['instance_name'])))
++ edge_conf['instance_name'])))
print("Key name: {}".format(edge_conf['key_name']))
print("Bucket name: {}".format(edge_conf['bucket_name']))
print("Shared bucket name: {}".format(edge_conf['shared_bucket_name']))
@@@ -302,7 -302,7 +302,8 @@@
res = {"hostname": edge_conf['instance_hostname'],
"public_ip": edge_conf['edge_public_ip'],
"ip": edge_conf['edge_private_ip'],
- "instance_id": dlab.meta_lib.get_instance_by_name(edge_conf['tag_name'], edge_conf['instance_name']),
- "instance_id": datalab.meta_lib.get_instance_by_name(edge_conf['tag_name'], edge_conf['instance_name']),
++ "instance_id": datalab.meta_lib.get_instance_by_name(edge_conf['tag_name'],
++ edge_conf['instance_name']),
"key_name": edge_conf['key_name'],
"user_own_bicket_name": edge_conf['bucket_name'],
"shared_bucket_name": edge_conf['shared_bucket_name'],
diff --cc infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
index c3ac2e4,20e8eed..e034ce0
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_configure_route_table.py
@@@ -22,10 -22,10 +22,9 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
-
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-
parser = argparse.ArgumentParser()
parser.add_argument('--vpc_id', type=str, default='')
parser.add_argument('--infra_tag_value', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/edge_start.py
index a9f856a,2bdcc10..68d2ef3
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_start.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_start.py
@@@ -21,15 -21,15 +21,11 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import sys
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import sys
++import json
import logging
import os
--import json
--
--
++import sys
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
diff --cc infrastructure-provisioning/src/general/scripts/aws/edge_status.py
index d8bd92e,596f16d..ed57891
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_status.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_status.py
@@@ -22,18 -22,18 +22,13 @@@
# ******************************************************************************
--import json
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import sys
- import time
- import os
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import sys
-import time
-import os
import logging
++import os
++import sys
import traceback
from fabric.api import *
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
os.environ['request_id'])
diff --cc infrastructure-provisioning/src/general/scripts/aws/edge_stop.py
index 3948781,6d8447b..1b460f6
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_stop.py
@@@ -21,14 -21,14 +21,11 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import sys
- import os
- import logging
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import sys
-import os
-import logging
import json
--
++import logging
++import os
++import sys
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
diff --cc infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index cc53b22,54815f8..5df07a1
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@@ -21,15 -21,15 +21,13 @@@
#
# ******************************************************************************
--import logging
++import argparse
++import datalab.meta_lib
import json
++import logging
++import os
import sys
import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
--import os
--import argparse
from fabric.api import *
parser = argparse.ArgumentParser()
@@@ -86,20 -86,20 +84,21 @@@ if __name__ == "__main__"
notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
# generating variables regarding EDGE proxy on Notebook instance
- instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
+ instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
++ notebook_config['instance_name'])
edge_instance_name = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'], notebook_config['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
- edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
- notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
++ edge_instance_name).get('Private')
+ notebook_config['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
++ edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
- edge_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
- edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
++ edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get(
++ 'Private')
except Exception as err:
- dlab.fab.append_result("Failed to generate variables dictionary.", str(err))
- dlab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
+ datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
try:
@@@ -312,9 -312,9 +311,9 @@@
try:
# generating output information
- ip_address = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
- dns_name = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
++ notebook_config['instance_name']).get('Private')
+ dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name'])
jupyter_dns_url = "http://" + dns_name + ":8888/{}/".format(notebook_config['exploratory_name'])
jupyter_notebook_access_url = "https://{}/{}/".format(notebook_config['edge_instance_hostname'],
@@@ -327,8 -327,8 +326,8 @@@
print("Instance name: {}".format(notebook_config['instance_name']))
print("Private DNS: {}".format(dns_name))
print("Private IP: {}".format(ip_address))
- print("Instance ID: {}".format(dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
+ print("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
++ notebook_config['instance_name'])))
print("Instance type: {}".format(notebook_config['instance_type']))
print("Key name: {}".format(notebook_config['key_name']))
print("User key name: {}".format(notebook_config['user_keyname']))
@@@ -348,8 -348,8 +347,8 @@@
with open("/root/result.json", 'w') as result:
res = {"hostname": dns_name,
"ip": ip_address,
- "instance_id": dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
+ "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
++ notebook_config['instance_name']),
"master_keyname": os.environ['conf_key_name'],
"notebook_name": notebook_config['instance_name'],
"notebook_image_name": notebook_config['notebook_image_name'],
diff --cc infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
index 1bb09b3,a6f7b14..4070c3a
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
@@@ -21,20 -21,20 +21,13 @@@
#
# ******************************************************************************
--import boto3
--from botocore.client import Config
--from fabric.api import *
import argparse
--import os
import sys
--import time
--from fabric.api import lcd
--from fabric.contrib.files import exists
--from fabvenv import virtualenv
- from dlab.notebook_lib import *
- from dlab.actions_lib import *
- from dlab.fab import *
- from dlab.common_lib import *
-from datalab.notebook_lib import *
+ from datalab.actions_lib import *
-from datalab.fab import *
+ from datalab.common_lib import *
++from datalab.fab import *
++from datalab.notebook_lib import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--bucket', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
index 37d102a,146a1db..0cfbd5c
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
@@@ -22,10 -22,10 +22,10 @@@
# ******************************************************************************
import argparse
--from fabric.api import *
import boto3
- from dlab.meta_lib import *
-from datalab.meta_lib import *
import os
++from datalab.meta_lib import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--bucket', type=str, default='')
@@@ -51,23 -51,23 +51,24 @@@ def configure_notebook(args)
files_dir = '/root/files/'
scripts_dir = '/root/scripts/'
put(templates_dir + 'sparkmagic_config_template.json', '/tmp/sparkmagic_config_template.json')
-- #put(templates_dir + 'pyspark_dataengine-service_template.json', '/tmp/pyspark_dataengine-service_template.json')
-- #put(templates_dir + 'r_dataengine-service_template.json', '/tmp/r_dataengine-service_template.json')
-- #put(templates_dir + 'toree_dataengine-service_template.json','/tmp/toree_dataengine-service_template.json')
++ # put(templates_dir + 'pyspark_dataengine-service_template.json', '/tmp/pyspark_dataengine-service_template.json')
++ # put(templates_dir + 'r_dataengine-service_template.json', '/tmp/r_dataengine-service_template.json')
++ # put(templates_dir + 'toree_dataengine-service_template.json','/tmp/toree_dataengine-service_template.json')
put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application),
'/tmp/jupyter_dataengine-service_create_configs.py')
-- #put(files_dir + 'toree_kernel.tar.gz', '/tmp/toree_kernel.tar.gz')
-- #put(templates_dir + 'toree_dataengine-service_templatev2.json', '/tmp/toree_dataengine-service_templatev2.json')
-- #put(templates_dir + 'run_template.sh', '/tmp/run_template.sh')
-- sudo('\cp /tmp/jupyter_dataengine-service_create_configs.py /usr/local/bin/jupyter_dataengine-service_create_configs.py')
++ # put(files_dir + 'toree_kernel.tar.gz', '/tmp/toree_kernel.tar.gz')
++ # put(templates_dir + 'toree_dataengine-service_templatev2.json', '/tmp/toree_dataengine-service_templatev2.json')
++ # put(templates_dir + 'run_template.sh', '/tmp/run_template.sh')
++ sudo(
++ '\cp /tmp/jupyter_dataengine-service_create_configs.py /usr/local/bin/jupyter_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/jupyter_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/dlab/')
- run('mkdir -p /tmp/dlab_libs/')
- local('scp -i {} /usr/lib/python2.7/dlab/* {}:/tmp/dlab_libs/'.format(args.keyfile, env.host_string))
- run('chmod a+x /tmp/dlab_libs/*')
- sudo('mv /tmp/dlab_libs/* /usr/lib/python2.7/dlab/')
+ sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ run('mkdir -p /tmp/datalab_libs/')
+ local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ run('chmod a+x /tmp/datalab_libs/*')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/dlab /usr/lib64/python2.7/dlab')
+ sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
if __name__ == "__main__":
diff --cc infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
index 950afa1,6b9d085..b7dd473
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyterlab_configure.py
@@@ -21,15 -21,15 +21,13 @@@
#
# ******************************************************************************
--import logging
++import argparse
++import datalab.meta_lib
import json
++import logging
++import os
import sys
import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
--import os
--import argparse
from fabric.api import *
parser = argparse.ArgumentParser()
@@@ -86,20 -86,20 +84,21 @@@ if __name__ == "__main__"
notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
# generating variables regarding EDGE proxy on Notebook instance
- instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
+ instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
++ notebook_config['instance_name'])
edge_instance_name = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'], notebook_config['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
- edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
- notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
++ edge_instance_name).get('Private')
+ notebook_config['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
++ edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
- edge_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
- edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
++ edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get(
++ 'Private')
except Exception as err:
- dlab.fab.append_result("Failed to generate variables dictionary.", str(err))
- dlab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
+ datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
try:
@@@ -110,11 -110,11 +109,11 @@@
notebook_config['initial_user'] = 'ec2-user'
notebook_config['sudo_group'] = 'wheel'
- logging.info('[CREATING DLAB SSH USER]')
- print('[CREATING DLAB SSH USER]')
- params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
- (instance_hostname, "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
- notebook_config['initial_user'], notebook_config['dlab_ssh_user'], notebook_config['sudo_group'])
+ logging.info('[CREATING DATALAB SSH USER]')
+ print('[CREATING DATALAB SSH USER]')
- params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
- (instance_hostname, "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
++ params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format \
++ (instance_hostname, "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
+ notebook_config['initial_user'], notebook_config['datalab_ssh_user'], notebook_config['sudo_group'])
try:
local("~/scripts/{}.py {}".format('create_ssh_user', params))
@@@ -148,8 -148,8 +147,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES TO JUPYTERLAB NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO JUPYTERLAB NOTEBOOK INSTANCE]')
-- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'])
++ params = "--hostname {} --keyfile {} --user {} --region {}". \
+ format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['aws_region'])
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
@@@ -331,9 -331,9 +330,9 @@@
try:
# generating output information
- ip_address = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
- dns_name = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
++ notebook_config['instance_name']).get('Private')
+ dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name'])
jupyter_dns_url = "http://" + dns_name + ":8888/{}/".format(notebook_config['exploratory_name'])
jupyter_notebook_acces_url = "http://{}/{}/".format(notebook_config['edge_instance_hostname'],
@@@ -346,8 -346,8 +345,8 @@@
print("Instance name: {}".format(notebook_config['instance_name']))
print("Private DNS: {}".format(dns_name))
print("Private IP: {}".format(ip_address))
- print("Instance ID: {}".format(dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
+ print("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
++ notebook_config['instance_name'])))
print("Instance type: {}".format(notebook_config['instance_type']))
print("Key name: {}".format(notebook_config['key_name']))
print("User key name: {}".format(notebook_config['user_keyname']))
@@@ -367,8 -367,8 +366,8 @@@
with open("/root/result.json", 'w') as result:
res = {"hostname": dns_name,
"ip": ip_address,
- "instance_id": dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
+ "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
++ notebook_config['instance_name']),
"master_keyname": os.environ['conf_key_name'],
"notebook_name": notebook_config['instance_name'],
"notebook_image_name": notebook_config['notebook_image_name'],
diff --cc infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index 9986ab2,4f435dd..e500845
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@@ -21,19 -21,19 +21,16 @@@
#
# ******************************************************************************
-import json
-import datalab.fab
-import datalab.actions_lib
++import boto3
+ import datalab.meta_lib
+import json
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
++import os
import sys
import time
--import os
import traceback
--import boto3
--import logging
from fabric.api import *
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id'])
@@@ -220,7 -220,7 +217,7 @@@
project_conf['endpoint_name'])
params = "--role_name {} --role_profile_name {} --policy_name {} --region {} --infra_tag_name {} " \
"--infra_tag_value {} --user_tag_value {}" \
-- .format(project_conf['edge_role_name'], project_conf['edge_role_profile_name'],
++ .format(project_conf['edge_role_name'], project_conf['edge_role_profile_name'],
project_conf['edge_policy_name'], os.environ['aws_region'], project_conf['tag_name'],
project_conf['service_base_name'], user_tag)
try:
@@@ -627,11 -627,11 +624,11 @@@
project_conf['edge_instance_name'])
try:
local("~/scripts/{}.py {}".format('common_create_instance', params))
- edge_instance = dlab.meta_lib.get_instance_by_name(project_conf['tag_name'],
- project_conf['edge_instance_name'])
+ edge_instance = datalab.meta_lib.get_instance_by_name(project_conf['tag_name'],
- project_conf['edge_instance_name'])
++ project_conf['edge_instance_name'])
if os.environ['edge_is_nat']:
try:
- dlab.actions_lib.modify_instance_sourcedescheck(edge_instance)
+ datalab.actions_lib.modify_instance_sourcedescheck(edge_instance)
except:
traceback.print_exc()
raise Exception
@@@ -652,8 -652,8 +649,8 @@@
try:
logging.info('[ASSOCIATING ELASTIC IP]')
print('[ASSOCIATING ELASTIC IP]')
- project_conf['edge_id'] = dlab.meta_lib.get_instance_by_name(project_conf['tag_name'],
- project_conf['edge_instance_name'])
+ project_conf['edge_id'] = datalab.meta_lib.get_instance_by_name(project_conf['tag_name'],
- project_conf['edge_instance_name'])
++ project_conf['edge_instance_name'])
try:
project_conf['elastic_ip'] = os.environ['edge_elastic_ip']
except:
@@@ -706,11 -706,11 +703,11 @@@
project_conf['edge_public_ip'])
except:
print("No Elastic IPs to release!")
- dlab.actions_lib.remove_ec2(project_conf['tag_name'], project_conf['edge_instance_name'])
- dlab.actions_lib.remove_all_iam_resources('notebook', project_conf['project_name'])
- dlab.actions_lib.remove_all_iam_resources('edge', project_conf['project_name'])
- dlab.actions_lib.remove_sgroups(project_conf['dataengine_instances_name'])
- dlab.actions_lib.remove_sgroups(project_conf['notebook_instance_name'])
- dlab.actions_lib.remove_sgroups(project_conf['edge_instance_name'])
- dlab.actions_lib.remove_s3('edge', project_conf['project_name'])
- sys.exit(1)
+ datalab.actions_lib.remove_ec2(project_conf['tag_name'], project_conf['edge_instance_name'])
+ datalab.actions_lib.remove_all_iam_resources('notebook', project_conf['project_name'])
+ datalab.actions_lib.remove_all_iam_resources('edge', project_conf['project_name'])
+ datalab.actions_lib.remove_sgroups(project_conf['dataengine_instances_name'])
+ datalab.actions_lib.remove_sgroups(project_conf['notebook_instance_name'])
+ datalab.actions_lib.remove_sgroups(project_conf['edge_instance_name'])
+ datalab.actions_lib.remove_s3('edge', project_conf['project_name'])
- sys.exit(1)
++ sys.exit(1)
diff --cc infrastructure-provisioning/src/general/scripts/aws/project_terminate.py
index 9243ff5,6d0db7f..6db396e
--- a/infrastructure-provisioning/src/general/scripts/aws/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_terminate.py
@@@ -21,23 -21,23 +21,23 @@@
#
# ******************************************************************************
-import json
-import datalab.fab
++import boto3
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import sys
-import time
-import os
-import traceback
+import json
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import sys
- import time
- import os
- import traceback
import logging
--import boto3
++import os
import requests
++import sys
++import traceback
--def terminate_edge_node(tag_name, project_name, tag_value, nb_sg, edge_sg, de_sg, emr_sg, endpoint_name, service_base_name):
++def terminate_edge_node(tag_name, project_name, tag_value, nb_sg, edge_sg, de_sg, emr_sg, endpoint_name,
++ service_base_name):
print('Terminating EMR cluster')
try:
- clusters_list = dlab.meta_lib.get_emr_list(tag_name)
+ clusters_list = datalab.meta_lib.get_emr_list(tag_name)
if clusters_list:
for cluster_id in clusters_list:
client = boto3.client('emr')
@@@ -149,8 -149,8 +149,8 @@@ if __name__ == "__main__"
sys.exit(1)
try:
- endpoint_id = dlab.meta_lib.get_instance_by_name(project_conf['tag_name'],
- project_conf['endpoint_instance_name'])
+ endpoint_id = datalab.meta_lib.get_instance_by_name(project_conf['tag_name'],
- project_conf['endpoint_instance_name'])
++ project_conf['endpoint_instance_name'])
print("Endpoint id: " + endpoint_id)
ec2 = boto3.client('ec2')
ec2.delete_tags(Resources=[endpoint_id], Tags=[{'Key': 'project_tag'}, {'Key': 'endpoint_tag'}])
diff --cc infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
index dd2a93c,ab4f6e3..07ec094
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
@@@ -21,15 -21,15 +21,13 @@@
#
# ******************************************************************************
--import logging
++import argparse
++import datalab.meta_lib
import json
++import logging
++import os
import sys
import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
--import os
--import argparse
from fabric.api import *
parser = argparse.ArgumentParser()
@@@ -87,21 -87,21 +85,22 @@@ if __name__ == "__main__"
notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
# generating variables regarding EDGE proxy on Notebook instance
- instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
+ instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
++ notebook_config['instance_name'])
edge_instance_name = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'], notebook_config['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
- edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
- notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
++ edge_instance_name).get('Private')
+ notebook_config['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
++ edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
- edge_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
- notebook_config['rstudio_pass'] = dlab.fab.id_generator()
- edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
++ edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get(
++ 'Private')
+ notebook_config['rstudio_pass'] = datalab.fab.id_generator()
except Exception as err:
- dlab.fab.append_result("Failed to generate variables dictionary.", str(err))
- dlab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
+ datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
try:
@@@ -150,8 -150,8 +149,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES TO R_STUDIO NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO R_STUDIO NOTEBOOK INSTANCE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['aws_region'],
edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -242,8 -242,8 +241,8 @@@
'instance_hostname': instance_hostname,
'tensor': False
}
-- params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'"\
- .format(edge_instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], 'rstudio',
++ params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'" \
+ .format(edge_instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], 'rstudio',
notebook_config['exploratory_name'], json.dumps(additional_info))
try:
local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
@@@ -291,9 -291,9 +290,9 @@@
try:
# generating output information
- ip_address = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
- dns_name = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
++ notebook_config['instance_name']).get('Private')
+ dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
rstudio_ip_url = "http://" + ip_address + ":8787/"
rstudio_dns_url = "http://" + dns_name + ":8787/"
rstudio_notebook_access_url = "https://{}/{}/".format(notebook_config['edge_instance_hostname'],
@@@ -306,8 -306,8 +305,8 @@@
print("Instance name: {}".format(notebook_config['instance_name']))
print("Private DNS: {}".format(dns_name))
print("Private IP: {}".format(ip_address))
- print("Instance ID: {}".format(dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
+ print("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
++ notebook_config['instance_name'])))
print("Instance type: {}".format(notebook_config['instance_type']))
print("Key name: {}".format(notebook_config['key_name']))
print("User key name: {}".format(notebook_config['user_keyname']))
@@@ -327,8 -327,8 +326,8 @@@
with open("/root/result.json", 'w') as result:
res = {"hostname": dns_name,
"ip": ip_address,
- "instance_id": dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
+ "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
++ notebook_config['instance_name']),
"master_keyname": os.environ['conf_key_name'],
"notebook_name": notebook_config['instance_name'],
"notebook_image_name": notebook_config['notebook_image_name'],
@@@ -337,13 -337,13 +336,13 @@@
{"description": "RStudio",
"url": rstudio_notebook_access_url},
{"description": "Ungit",
-- "url": rstudio_ungit_access_url}#,
-- #{"description": "RStudio (via tunnel)",
++ "url": rstudio_ungit_access_url} # ,
++ # {"description": "RStudio (via tunnel)",
# "url": rstudio_ip_url},
-- #{"description": "Ungit (via tunnel)",
++ # {"description": "Ungit (via tunnel)",
# "url": ungit_ip_url}
],
- "exploratory_user": notebook_config['dlab_ssh_user'],
+ "exploratory_user": notebook_config['datalab_ssh_user'],
"exploratory_pass": notebook_config['rstudio_pass']}
result.write(json.dumps(res))
except Exception as err:
diff --cc infrastructure-provisioning/src/general/scripts/aws/rstudio_dataengine-service_create_configs.py
index f82c0f0,443d147..d4f38e7
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_dataengine-service_create_configs.py
@@@ -21,20 -21,20 +21,14 @@@
#
# ******************************************************************************
--import boto3
--from botocore.client import Config
--from fabric.api import *
import argparse
import os
import sys
--import time
--from fabric.api import lcd
--from fabric.contrib.files import exists
--from fabvenv import virtualenv
- from dlab.notebook_lib import *
- from dlab.actions_lib import *
- from dlab.fab import *
- from dlab.common_lib import *
-from datalab.notebook_lib import *
+ from datalab.actions_lib import *
-from datalab.fab import *
+ from datalab.common_lib import *
++from datalab.fab import *
++from datalab.notebook_lib import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--bucket', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
index 4393b21,da38168..ee7800b
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_install_dataengine-service_kernels.py
@@@ -22,10 -22,10 +22,8 @@@
# ******************************************************************************
import argparse
-from fabric.api import *
-import boto3
+ from datalab.meta_lib import *
-import os
+from fabric.api import *
- import boto3
- from dlab.meta_lib import *
- import os
parser = argparse.ArgumentParser()
parser.add_argument('--bucket', type=str, default='')
@@@ -50,15 -50,15 +48,16 @@@ def configure_notebook(args)
scripts_dir = '/root/scripts/'
put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application),
'/tmp/rstudio_dataengine-service_create_configs.py')
-- sudo('\cp /tmp/rstudio_dataengine-service_create_configs.py /usr/local/bin/rstudio_dataengine-service_create_configs.py')
++ sudo(
++ '\cp /tmp/rstudio_dataengine-service_create_configs.py /usr/local/bin/rstudio_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/rstudio_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/dlab/')
- run('mkdir -p /tmp/dlab_libs/')
- local('scp -i {} /usr/lib/python2.7/dlab/* {}:/tmp/dlab_libs/'.format(args.keyfile, env.host_string))
- run('chmod a+x /tmp/dlab_libs/*')
- sudo('mv /tmp/dlab_libs/* /usr/lib/python2.7/dlab/')
+ sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ run('mkdir -p /tmp/datalab_libs/')
+ local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ run('chmod a+x /tmp/datalab_libs/*')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/dlab /usr/lib64/python2.7/dlab')
+ sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
if __name__ == "__main__":
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_associate_elastic_ip.py
index 55bb37d,3963864..bc08f5f
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_associate_elastic_ip.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_associate_elastic_ip.py
@@@ -21,12 -21,12 +21,12 @@@
#
# ******************************************************************************
- import sys
- from dlab.fab import *
- from dlab.meta_lib import *
- from dlab.actions_lib import *
- import os
+import argparse
++import os
+ import sys
++from datalab.actions_lib import *
+ from datalab.fab import *
+ from datalab.meta_lib import *
-from datalab.actions_lib import *
-import os
-import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--elastic_ip', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
index 759c417,fc4bc36..128fe0a
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
@@@ -21,16 -21,16 +21,16 @@@
#
# ******************************************************************************
-import logging
-import sys
-import os
-from fabric.api import *
-import datalab.ssn_lib
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import traceback
++import datalab.ssn_lib
+ import json
+import logging
- import sys
+import os
- from fabric.api import *
- import dlab.ssn_lib
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import sys
+import traceback
- import json
++from fabric.api import *
if __name__ == "__main__":
local_log_filename = "{}_{}.log".format(os.environ['conf_resource'], os.environ['request_id'])
@@@ -44,30 -44,30 +44,30 @@@
def clear_resources():
if ssn_conf['domain_created']:
- dlab.actions_lib.remove_route_53_record(os.environ['ssn_hosted_zone_id'],
- os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'])
- dlab.actions_lib.remove_ec2(ssn_conf['tag_name'], ssn_conf['instance_name'])
- dlab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
- dlab.actions_lib.remove_s3(ssn_conf['instance'])
+ datalab.actions_lib.remove_route_53_record(os.environ['ssn_hosted_zone_id'],
- os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'])
++ os.environ['ssn_hosted_zone_name'],
++ os.environ['ssn_subdomain'])
+ datalab.actions_lib.remove_ec2(ssn_conf['tag_name'], ssn_conf['instance_name'])
+ datalab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
+ datalab.actions_lib.remove_s3(ssn_conf['instance'])
if ssn_conf['pre_defined_sg']:
- dlab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
+ datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
if ssn_conf['pre_defined_subnet']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
try:
logging.info('[DERIVING NAMES]')
@@@ -102,7 -102,7 +102,8 @@@
raise KeyError
except KeyError:
ssn_conf['tag'] = {"Key": ssn_conf['tag_name'], "Value": "{}-subnet".format(ssn_conf['service_base_name'])}
- os.environ['aws_vpc_id'] = dlab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'], ssn_conf['service_base_name'])
- os.environ['aws_vpc_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'], ssn_conf['service_base_name'])
++ os.environ['aws_vpc_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'],
++ ssn_conf['service_base_name'])
ssn_conf['pre_defined_vpc'] = True
try:
if os.environ['aws_subnet_id'] == '':
@@@ -116,15 -116,15 +117,15 @@@
raise KeyError
except KeyError:
ssn_conf['tag'] = {"Key": ssn_conf['tag2_name'], "Value": "{}-subnet".format(ssn_conf['service_base_name'])}
- os.environ['aws_vpc2_id'] = dlab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
- ssn_conf['service_base_name'])
+ os.environ['aws_vpc2_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
ssn_conf['pre_defined_vpc2'] = True
try:
if os.environ['conf_duo_vpc_enable'] == 'true' and not os.environ['aws_peering_id']:
raise KeyError
except KeyError:
- os.environ['aws_peering_id'] = dlab.meta_lib.get_peering_by_tag(ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
+ os.environ['aws_peering_id'] = datalab.meta_lib.get_peering_by_tag(ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
ssn_conf['pre_defined_peering'] = True
try:
if os.environ['aws_security_groups_ids'] == '':
@@@ -172,9 -172,9 +173,9 @@@
ssn_conf['tag_name'], ssn_conf['instance_name']).get('Private'))
if ssn_conf['network_type'] == 'public':
ssn_conf['step_cert_sans'] += ' --san {0} --san {1}'.format(
- dlab.meta_lib.get_instance_hostname(ssn_conf['tag_name'], ssn_conf['instance_name']),
- dlab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
- ssn_conf['instance_name']).get('Public'))
+ datalab.meta_lib.get_instance_hostname(ssn_conf['tag_name'], ssn_conf['instance_name']),
+ datalab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
- ssn_conf['instance_name']).get('Public'))
++ ssn_conf['instance_name']).get('Public'))
else:
ssn_conf['step_cert_sans'] = ''
@@@ -216,16 -216,16 +217,16 @@@
logging.info('[CONFIGURE SSN INSTANCE]')
print('[CONFIGURE SSN INSTANCE]')
additional_config = {"nginx_template_dir": "/root/templates/", "service_base_name":
-- ssn_conf['service_base_name'],
++ ssn_conf['service_base_name'],
"security_group_id": os.environ['aws_security_groups_ids'],
"vpc_id": os.environ['aws_vpc_id'], "subnet_id": os.environ['aws_subnet_id'],
"admin_key": os.environ['conf_key_name']}
- params = "--hostname {} --keyfile {} --additional_config '{}' --os_user {} --dlab_path {} " \
+ params = "--hostname {} --keyfile {} --additional_config '{}' --os_user {} --datalab_path {} " \
"--tag_resource_id {} --step_cert_sans '{}' ".format(
-- ssn_conf['instance_hostname'],
-- "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
- json.dumps(additional_config), ssn_conf['dlab_ssh_user'], os.environ['ssn_dlab_path'],
- json.dumps(additional_config), ssn_conf['datalab_ssh_user'], os.environ['ssn_datalab_path'],
-- os.environ['conf_tag_resource_id'], ssn_conf['step_cert_sans'])
++ ssn_conf['instance_hostname'],
++ "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
++ json.dumps(additional_config), ssn_conf['datalab_ssh_user'], os.environ['ssn_datalab_path'],
++ os.environ['conf_tag_resource_id'], ssn_conf['step_cert_sans'])
try:
local("~/scripts/{}.py {}".format('configure_ssn_node', params))
@@@ -274,8 -274,8 +275,8 @@@
cloud_params = [
{
'key': 'KEYCLOAK_REDIRECT_URI',
- 'value': "https://{0}/".format(dlab.meta_lib.get_instance_hostname(ssn_conf['tag_name'],
- ssn_conf['instance_name']))
+ 'value': "https://{0}/".format(datalab.meta_lib.get_instance_hostname(ssn_conf['tag_name'],
- ssn_conf['instance_name']))
++ ssn_conf['instance_name']))
},
{
'key': 'KEYCLOAK_REALM_NAME',
@@@ -658,8 -658,8 +659,8 @@@
with open("/root/result.json", 'w') as f:
res = {"service_base_name": ssn_conf['service_base_name'],
"instance_name": ssn_conf['instance_name'],
- "instance_hostname": dlab.meta_lib.get_instance_hostname(ssn_conf['tag_name'],
- ssn_conf['instance_name']),
+ "instance_hostname": datalab.meta_lib.get_instance_hostname(ssn_conf['tag_name'],
- ssn_conf['instance_name']),
++ ssn_conf['instance_name']),
"role_name": ssn_conf['role_name'],
"role_profile_name": ssn_conf['role_profile_name'],
"policy_name": ssn_conf['policy_name'],
@@@ -673,8 -673,8 +674,8 @@@
f.write(json.dumps(res))
print('Upload response file')
-- params = "--instance_name {} --local_log_filepath {} --os_user {} --instance_hostname {}".\
- format(ssn_conf['instance_name'], local_log_filepath, ssn_conf['dlab_ssh_user'],
++ params = "--instance_name {} --local_log_filepath {} --os_user {} --instance_hostname {}". \
+ format(ssn_conf['instance_name'], local_log_filepath, ssn_conf['datalab_ssh_user'],
ssn_conf['instance_hostname'])
local("~/scripts/{}.py {}".format('upload_response_file', params))
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_create_endpoint.py
index f44c6d9,7d751be..f54b89f
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_create_endpoint.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_create_endpoint.py
@@@ -22,12 -22,12 +22,13 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
++import boto3
++import botocore
+import sys
- import boto3, botocore
- from dlab.ssn_lib import *
+import time
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
-import boto3, botocore
+ from datalab.ssn_lib import *
-import time
parser = argparse.ArgumentParser()
parser.add_argument('--vpc_id', type=str, default='')
@@@ -36,7 -36,7 +37,6 @@@ parser.add_argument('--infra_tag_name'
parser.add_argument('--infra_tag_value', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
tag = {"Key": args.infra_tag_name, "Value": args.infra_tag_value}
waiter = time.sleep(10)
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
index 3ab99dd,fa3348e..6042606
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_create_vpc.py
@@@ -22,10 -22,10 +22,9 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
-
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-
parser = argparse.ArgumentParser()
parser.add_argument('--vpc', type=str, default='')
parser.add_argument('--secondary', dest='secondary', action='store_true')
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_finalize.py
index ee6ac9d,0586714..e85379e
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_finalize.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_finalize.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import boto3
import argparse
++import boto3
import sys
- from dlab.ssn_lib import *
-
+ from datalab.ssn_lib import *
-
parser = argparse.ArgumentParser()
parser.add_argument('--key_id', type=str, default='')
args = parser.parse_args()
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
index 45c65f2,109aca8..866d903
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_prepare.py
@@@ -21,17 -21,17 +21,13 @@@
#
# ******************************************************************************
++import datalab.meta_lib
++import json
import logging
--import sys
import os
--from fabric.api import *
- import dlab.ssn_lib
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.ssn_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
++import sys
import traceback
--import json
--
++from fabric.api import *
if __name__ == "__main__":
local_log_filename = "{}_{}.log".format(os.environ['conf_resource'], os.environ['request_id'])
@@@ -111,10 -111,10 +107,10 @@@
except:
traceback.print_exc()
raise Exception
- os.environ['aws_vpc_id'] = dlab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
+ os.environ['aws_vpc_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
except Exception as err:
- dlab.fab.append_result("Failed to create VPC", str(err))
+ datalab.fab.append_result("Failed to create VPC", str(err))
sys.exit(1)
ssn_conf['allowed_vpc_cidr_ip_ranges'] = list()
@@@ -137,15 -137,15 +133,15 @@@
except:
traceback.print_exc()
raise Exception
- os.environ['aws_vpc2_id'] = dlab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
- ssn_conf['service_base_name'])
+ os.environ['aws_vpc2_id'] = datalab.meta_lib.get_vpc_by_tag(ssn_conf['tag2_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
except Exception as err:
- dlab.fab.append_result("Failed to create secondary VPC.", str(err))
+ datalab.fab.append_result("Failed to create secondary VPC.", str(err))
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
sys.exit(1)
try:
@@@ -167,25 -167,25 +163,25 @@@
raise Exception
with open('/tmp/ssn_subnet_id', 'r') as f:
os.environ['aws_subnet_id'] = f.read()
- dlab.actions_lib.enable_auto_assign_ip(os.environ['aws_subnet_id'])
+ datalab.actions_lib.enable_auto_assign_ip(os.environ['aws_subnet_id'])
except Exception as err:
- dlab.fab.append_result("Failed to create Subnet.", str(err))
+ datalab.fab.append_result("Failed to create Subnet.", str(err))
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
try:
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
except:
print("Subnet hasn't been created.")
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
try:
@@@ -195,29 -195,29 +191,29 @@@
try:
logging.info('[CREATE PEERING CONNECTION]')
print('[CREATE PEERING CONNECTION]')
- os.environ['aws_peering_id'] = dlab.actions_lib.create_peering_connection(
+ os.environ['aws_peering_id'] = datalab.actions_lib.create_peering_connection(
os.environ['aws_vpc_id'], os.environ['aws_vpc2_id'], ssn_conf['service_base_name'])
print('PEERING CONNECTION ID:' + os.environ['aws_peering_id'])
- dlab.actions_lib.create_route_by_id(os.environ['aws_subnet_id'], os.environ['aws_vpc_id'],
- os.environ['aws_peering_id'],
- dlab.meta_lib.get_cidr_by_vpc(os.environ['aws_vpc2_id']))
+ datalab.actions_lib.create_route_by_id(os.environ['aws_subnet_id'], os.environ['aws_vpc_id'],
- os.environ['aws_peering_id'],
- datalab.meta_lib.get_cidr_by_vpc(os.environ['aws_vpc2_id']))
++ os.environ['aws_peering_id'],
++ datalab.meta_lib.get_cidr_by_vpc(os.environ['aws_vpc2_id']))
except Exception as err:
- dlab.fab.append_result("Failed to create peering connection.", str(err))
+ datalab.fab.append_result("Failed to create peering connection.", str(err))
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
try:
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
except:
print("Subnet hasn't been created.")
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
try:
@@@ -283,21 -283,21 +279,21 @@@
with open('/tmp/ssn_sg_id', 'r') as f:
os.environ['aws_security_groups_ids'] = f.read()
except Exception as err:
- dlab.gab_lib.append_result("Failed creating security group for SSN.", str(err))
+ datalab.gab_lib.append_result("Failed creating security group for SSN.", str(err))
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
try:
@@@ -314,24 -314,24 +310,24 @@@
traceback.print_exc()
raise Exception
except Exception as err:
- dlab.fab.append_result("Unable to create roles.", str(err))
+ datalab.fab.append_result("Unable to create roles.", str(err))
if ssn_conf['pre_defined_sg']:
- dlab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
+ datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
if ssn_conf['pre_defined_subnet']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
try:
@@@ -345,25 -345,25 +341,25 @@@
traceback.print_exc()
raise Exception
except Exception as err:
- dlab.fab.append_result("Unable to create an endpoint.", str(err))
- dlab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
+ datalab.fab.append_result("Unable to create an endpoint.", str(err))
+ datalab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
if ssn_conf['pre_defined_sg']:
- dlab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
+ datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
if ssn_conf['pre_defined_subnet']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
if os.environ['conf_duo_vpc_enable'] == 'true':
@@@ -379,25 -379,25 +375,25 @@@
traceback.print_exc()
raise Exception
except Exception as err:
- dlab.fab.append_result("Unable to create secondary endpoint.", str(err))
- dlab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
+ datalab.fab.append_result("Unable to create secondary endpoint.", str(err))
+ datalab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
if ssn_conf['pre_defined_sg']:
- dlab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
+ datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
if ssn_conf['pre_defined_subnet']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
try:
@@@ -416,27 -416,27 +412,27 @@@
traceback.print_exc()
raise Exception
except Exception as err:
- dlab.fab.append_result("Unable to create ssn instance.", str(err))
- dlab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
- dlab.actions_lib.remove_s3(ssn_conf['instance'])
+ datalab.fab.append_result("Unable to create ssn instance.", str(err))
+ datalab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
+ datalab.actions_lib.remove_s3(ssn_conf['instance'])
if ssn_conf['pre_defined_sg']:
- dlab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
+ datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
if ssn_conf['pre_defined_subnet']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
if ssn_conf['network_type'] == 'public':
@@@ -456,72 -456,72 +452,72 @@@
traceback.print_exc()
raise Exception
except Exception as err:
- dlab.fab.append_result("Failed to associate elastic ip.", str(err))
- dlab.actions_lib.remove_ec2(ssn_conf['tag_name'], ssn_conf['instance_name'])
- dlab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
- dlab.actions_lib.remove_s3(ssn_conf['instance'])
+ datalab.fab.append_result("Failed to associate elastic ip.", str(err))
+ datalab.actions_lib.remove_ec2(ssn_conf['tag_name'], ssn_conf['instance_name'])
+ datalab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
+ datalab.actions_lib.remove_s3(ssn_conf['instance'])
if ssn_conf['pre_defined_sg']:
- dlab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
+ datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
if ssn_conf['pre_defined_subnet']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
if ssn_conf['network_type'] == 'private':
- ssn_conf['instance_ip'] = dlab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
- ssn_conf['instance_name']).get('Private')
+ ssn_conf['instance_ip'] = datalab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
- ssn_conf['instance_name']).get('Private')
++ ssn_conf['instance_name']).get('Private')
else:
- ssn_conf['instance_ip'] = dlab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
- ssn_conf['instance_name']).get('Public')
+ ssn_conf['instance_ip'] = datalab.meta_lib.get_instance_ip_address(ssn_conf['tag_name'],
- ssn_conf['instance_name']).get('Public')
++ ssn_conf['instance_name']).get('Public')
if 'ssn_hosted_zone_id' in os.environ and 'ssn_hosted_zone_name' in os.environ and 'ssn_subdomain' in os.environ:
try:
logging.info('[CREATING ROUTE53 RECORD]')
print('[CREATING ROUTE53 RECORD]')
try:
- dlab.actions_lib.create_route_53_record(os.environ['ssn_hosted_zone_id'],
- os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'], ssn_conf['instance_ip'])
+ datalab.actions_lib.create_route_53_record(os.environ['ssn_hosted_zone_id'],
- os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'], ssn_conf['instance_ip'])
++ os.environ['ssn_hosted_zone_name'],
++ os.environ['ssn_subdomain'], ssn_conf['instance_ip'])
except:
traceback.print_exc()
raise Exception
except Exception as err:
- dlab.fab.append_result("Failed to create route53 record.", str(err))
- dlab.actions_lib.remove_route_53_record(os.environ['ssn_hosted_zone_id'],
- os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'])
- dlab.actions_lib.remove_ec2(ssn_conf['tag_name'], ssn_conf['instance_name'])
- dlab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
- dlab.actions_lib.remove_s3(ssn_conf['instance'])
+ datalab.fab.append_result("Failed to create route53 record.", str(err))
+ datalab.actions_lib.remove_route_53_record(os.environ['ssn_hosted_zone_id'],
- os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'])
++ os.environ['ssn_hosted_zone_name'],
++ os.environ['ssn_subdomain'])
+ datalab.actions_lib.remove_ec2(ssn_conf['tag_name'], ssn_conf['instance_name'])
+ datalab.actions_lib.remove_all_iam_resources(ssn_conf['instance'])
+ datalab.actions_lib.remove_s3(ssn_conf['instance'])
if ssn_conf['pre_defined_sg']:
- dlab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
+ datalab.actions_lib.remove_sgroups(ssn_conf['tag_name'])
if ssn_conf['pre_defined_subnet']:
- dlab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
- dlab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
+ datalab.actions_lib.remove_internet_gateways(os.environ['aws_vpc_id'], ssn_conf['tag_name'],
- ssn_conf['service_base_name'])
++ ssn_conf['service_base_name'])
+ datalab.actions_lib.remove_subnets(ssn_conf['subnet_name'])
if ssn_conf['pre_defined_vpc']:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
- dlab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc_id'])
if ssn_conf['pre_defined_vpc2']:
- dlab.actions_lib.remove_peering('*')
+ datalab.actions_lib.remove_peering('*')
try:
- dlab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_vpc_endpoints(os.environ['aws_vpc2_id'])
except:
print("There are no VPC Endpoints")
- dlab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
- dlab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
+ datalab.actions_lib.remove_route_tables(ssn_conf['tag2_name'], True)
+ datalab.actions_lib.remove_vpc(os.environ['aws_vpc2_id'])
sys.exit(1)
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_terminate.py
index 975e8d3,c6e70bb..13c8434
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate.py
@@@ -21,16 -21,16 +21,13 @@@
#
# ******************************************************************************
--import sys
--import os
++import datalab.ssn_lib
++import json
import logging
++import os
++import sys
import traceback
from fabric.api import *
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import dlab.ssn_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
-import datalab.ssn_lib
--import json
if __name__ == "__main__":
local_log_filename = "{}_{}.log".format(os.environ['conf_resource'], os.environ['request_id'])
@@@ -40,13 -40,13 +37,13 @@@
filename=local_log_filepath)
# generating variables dictionary
if 'aws_access_key' in os.environ and 'aws_secret_access_key' in os.environ:
- dlab.actions_lib.create_aws_config_files(generate_full_config=True)
+ datalab.actions_lib.create_aws_config_files(generate_full_config=True)
else:
- dlab.actions_lib.create_aws_config_files()
+ datalab.actions_lib.create_aws_config_files()
print('Generating infrastructure names and tags')
ssn_conf = dict()
- ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] = dlab.fab.replace_multi_symbols(
- os.environ['conf_service_base_name'][:20], '-', True)
+ ssn_conf['service_base_name'] = os.environ['conf_service_base_name'] = datalab.fab.replace_multi_symbols(
- os.environ['conf_service_base_name'][:20], '-', True)
++ os.environ['conf_service_base_name'][:20], '-', True)
ssn_conf['tag_name'] = ssn_conf['service_base_name'] + '-tag'
ssn_conf['edge_sg'] = ssn_conf['service_base_name'] + "*" + '-edge'
ssn_conf['nb_sg'] = ssn_conf['service_base_name'] + "*" + '-nb'
diff --cc infrastructure-provisioning/src/general/scripts/aws/ssn_terminate_aws_resources.py
index 27b5913,d0a7eef..f2f992f
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate_aws_resources.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_terminate_aws_resources.py
@@@ -21,14 -21,14 +21,11 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import dlab.ssn_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
-import datalab.ssn_lib
--import boto3
import argparse
--import sys
++import boto3
++import datalab.ssn_lib
import os
++import sys
parser = argparse.ArgumentParser()
parser.add_argument('--tag_name', type=str)
@@@ -78,8 -78,8 +75,8 @@@ if __name__ == "__main__"
if 'ssn_hosted_zone_id' in os.environ and 'ssn_hosted_zone_name' in os.environ and 'ssn_subdomain' in os.environ:
print("Removing Route53 records")
- dlab.actions_lib.remove_route_53_record(os.environ['ssn_hosted_zone_id'], os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'])
+ datalab.actions_lib.remove_route_53_record(os.environ['ssn_hosted_zone_id'], os.environ['ssn_hosted_zone_name'],
- os.environ['ssn_subdomain'])
++ os.environ['ssn_subdomain'])
print("Removing security groups")
try:
diff --cc infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
index 6baaf45,5028e4c..3d69022
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
@@@ -21,15 -21,15 +21,12 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import traceback
-import datalab.fab
-import datalab.actions_lib
++import argparse
+ import datalab.meta_lib
+import json
- import sys
- import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
--import argparse
++import sys
import traceback
from fabric.api import *
@@@ -37,7 -37,7 +34,6 @@@ parser = argparse.ArgumentParser(
parser.add_argument('--uuid', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
instance_class = 'notebook'
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -88,21 -88,21 +84,22 @@@
notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
# generating variables regarding EDGE proxy on Notebook instance
- instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
+ instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
++ notebook_config['instance_name'])
edge_instance_name = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'], notebook_config['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
- edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
- notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
++ edge_instance_name).get('Private')
+ notebook_config['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
++ edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
- edge_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
- notebook_config['rstudio_pass'] = dlab.fab.id_generator()
- edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
++ edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get(
++ 'Private')
+ notebook_config['rstudio_pass'] = datalab.fab.id_generator()
except Exception as err:
- dlab.fab.append_result("Failed to generate variables dictionary.", str(err))
- dlab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
+ datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
try:
@@@ -151,8 -151,8 +148,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES TO TENSORFLOW-RSTUDIO NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO TENSORFLOW-RSTUDIO NOTEBOOK INSTANCE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['aws_region'],
edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -243,8 -243,8 +240,8 @@@
'instance_hostname': instance_hostname,
'tensor': True
}
-- params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'"\
- .format(edge_instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], 'rstudio',
++ params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'" \
+ .format(edge_instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], 'rstudio',
notebook_config['exploratory_name'], json.dumps(additional_info))
try:
local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
@@@ -293,9 -293,9 +290,9 @@@
try:
# generating output information
- ip_address = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
- dns_name = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
++ notebook_config['instance_name']).get('Private')
+ dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
tensorboard_url = "http://" + ip_address + ":6006/"
rstudio_ip_url = "http://" + ip_address + ":8787/"
rstudio_dns_url = "http://" + dns_name + ":8787/"
@@@ -311,8 -311,8 +308,8 @@@
print("Instance name: {}".format(notebook_config['instance_name']))
print("Private DNS: {}".format(dns_name))
print("Private IP: {}".format(ip_address))
- print("Instance ID: {}".format(dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
+ print("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
++ notebook_config['instance_name'])))
print("Instance type: {}".format(notebook_config['instance_type']))
print("Key name: {}".format(notebook_config['key_name']))
print("User key name: {}".format(notebook_config['user_keyname']))
@@@ -334,8 -334,8 +331,8 @@@
with open("/root/result.json", 'w') as result:
res = {"hostname": dns_name,
"ip": ip_address,
- "instance_id": dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
+ "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
++ notebook_config['instance_name']),
"master_keyname": os.environ['conf_key_name'],
"tensorboard_log_dir": "/var/log/tensorboard",
"notebook_name": notebook_config['instance_name'],
@@@ -347,15 -347,15 +344,15 @@@
{"description": "TensorBoard",
"url": tensorboard_access_url},
{"description": "Ungit",
-- "url": rstudio_ungit_access_url}#,
-- #{"description": "RStudio (via tunnel)",
++ "url": rstudio_ungit_access_url} # ,
++ # {"description": "RStudio (via tunnel)",
# "url": rstudio_ip_url},
-- #{"description": "TensorBoard (via tunnel)",
++ # {"description": "TensorBoard (via tunnel)",
# "url": tensorboard_url},
-- #{"description": "Ungit (via tunnel)",
++ # {"description": "Ungit (via tunnel)",
# "url": ungit_ip_url}
],
- "exploratory_user": notebook_config['dlab_ssh_user'],
+ "exploratory_user": notebook_config['datalab_ssh_user'],
"exploratory_pass": notebook_config['rstudio_pass']}
result.write(json.dumps(res))
except Exception as err:
diff --cc infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
index 3cf3a46,fc4c65a..cf7ed18
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
@@@ -21,15 -21,15 +21,12 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import traceback
-import datalab.fab
-import datalab.actions_lib
++import argparse
+ import datalab.meta_lib
+import json
- import sys
- import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
--import argparse
++import sys
import traceback
from fabric.api import *
@@@ -37,7 -37,7 +34,6 @@@ parser = argparse.ArgumentParser(
parser.add_argument('--uuid', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
instance_class = 'notebook'
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -88,20 -88,20 +84,21 @@@
notebook_config['tag_name'], notebook_config['instance_name']).get('Private')
# generating variables regarding EDGE proxy on Notebook instance
- instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
+ instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
++ notebook_config['instance_name'])
edge_instance_name = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'], notebook_config['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
- edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
- notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
++ edge_instance_name).get('Private')
+ notebook_config['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
++ edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
- edge_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
- edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
++ edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get(
++ 'Private')
except Exception as err:
- dlab.fab.append_result("Failed to generate variables dictionary.", str(err))
- dlab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
+ datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
try:
@@@ -150,8 -150,8 +147,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['aws_region'],
edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -239,8 -239,8 +236,8 @@@
'instance_hostname': instance_hostname,
'tensor': True
}
-- params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'"\
- .format(edge_instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], 'jupyter',
++ params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'" \
+ .format(edge_instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], 'jupyter',
notebook_config['exploratory_name'], json.dumps(additional_info))
try:
local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
@@@ -288,9 -288,9 +285,9 @@@
sys.exit(1)
try:
# generating output information
- ip_address = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
- dns_name = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
++ notebook_config['instance_name']).get('Private')
+ dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
tensorboard_url = "http://" + ip_address + ":6006/"
jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name'])
jupyter_notebook_access_url = "https://{}/{}/".format(notebook_config['edge_instance_hostname'],
@@@ -305,8 -305,8 +302,8 @@@
print("Instance name: {}".format(notebook_config['instance_name']))
print("Private DNS: {}".format(dns_name))
print("Private IP: {}".format(ip_address))
- print("Instance ID: {}".format(dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
+ print("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
++ notebook_config['instance_name'])))
print("Instance type: {}".format(notebook_config['instance_type']))
print("Key name: {}".format(notebook_config['key_name']))
print("User key name: {}".format(notebook_config['user_keyname']))
@@@ -325,8 -325,8 +322,8 @@@
with open("/root/result.json", 'w') as result:
res = {"hostname": dns_name,
"ip": ip_address,
- "instance_id": dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
+ "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
++ notebook_config['instance_name']),
"master_keyname": os.environ['conf_key_name'],
"tensorboard_log_dir": "/var/log/tensorboard",
"notebook_name": notebook_config['instance_name'],
diff --cc infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
index dbdae70,3cef609..42d1e42
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
@@@ -21,15 -21,15 +21,12 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import traceback
-import datalab.fab
-import datalab.actions_lib
++import argparse
+ import datalab.meta_lib
+import json
- import sys
- import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
--import argparse
++import sys
import traceback
from fabric.api import *
@@@ -37,7 -37,7 +34,6 @@@ parser = argparse.ArgumentParser(
parser.add_argument('--uuid', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
instance_class = 'notebook'
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -94,20 -94,20 +90,21 @@@
else:
notebook_config['endpoint_url'] = 'https://s3-{}.amazonaws.com'.format(notebook_config['region'])
# generating variables regarding EDGE proxy on Notebook instance
- instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
+ instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name'])
++ notebook_config['instance_name'])
edge_instance_name = '{}-{}-{}-edge'.format(notebook_config['service_base_name'],
notebook_config['project_name'], notebook_config['endpoint_name'])
- edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
- edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
- notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
+ edge_instance_hostname = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- edge_instance_name).get('Private')
++ edge_instance_name).get('Private')
+ notebook_config['edge_instance_hostname'] = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- edge_instance_name)
++ edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
- edge_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
- edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
++ edge_ip = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get(
++ 'Private')
except Exception as err:
- dlab.fab.append_result("Failed to generate variables dictionary.", str(err))
- dlab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
+ datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
+ datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
try:
@@@ -175,8 -175,8 +172,9 @@@
logging.info('[CONFIGURE ZEPPELIN NOTEBOOK INSTANCE]')
print('[CONFIGURE ZEPPELIN NOTEBOOK INSTANCE]')
additional_config = {"frontend_hostname": edge_instance_hostname,
- "backend_hostname": dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name']),
+ "backend_hostname": datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
- notebook_config['instance_name']),
++ notebook_config[
++ 'instance_name']),
"backend_port": "8080",
"nginx_template_dir": "/root/templates/"}
params = "--hostname {0} --instance_name {1} " \
@@@ -189,7 -189,7 +187,8 @@@
"--r_mirror {14} --endpoint_url {15} " \
"--ip_address {16} --exploratory_name {17} --edge_ip {18}" \
.format(instance_hostname, notebook_config['instance_name'], keyfile_name, os.environ['aws_region'],
- json.dumps(additional_config), notebook_config['dlab_ssh_user'], os.environ['notebook_spark_version'],
- json.dumps(additional_config), notebook_config['datalab_ssh_user'], os.environ['notebook_spark_version'],
++ json.dumps(additional_config), notebook_config['datalab_ssh_user'],
++ os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], edge_instance_hostname, '3128',
os.environ['notebook_zeppelin_version'], os.environ['notebook_scala_version'],
os.environ['notebook_livy_version'], os.environ['notebook_multiple_clusters'],
@@@ -261,8 -261,8 +260,8 @@@
'instance_hostname': instance_hostname,
'tensor': False
}
-- params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'"\
- .format(edge_instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], 'zeppelin',
++ params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'" \
+ .format(edge_instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], 'zeppelin',
notebook_config['exploratory_name'], json.dumps(additional_info))
try:
local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
@@@ -310,9 -310,9 +309,9 @@@
sys.exit(1)
try:
# generating output information
- ip_address = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
- dns_name = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
+ ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
- notebook_config['instance_name']).get('Private')
++ notebook_config['instance_name']).get('Private')
+ dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
zeppelin_ip_url = "http://" + ip_address + ":8080/"
zeppelin_dns_url = "http://" + dns_name + ":8080/"
zeppelin_notebook_access_url = "https://{}/{}/".format(notebook_config['edge_instance_hostname'],
@@@ -325,8 -325,8 +324,8 @@@
print("Instance name: {}".format(notebook_config['instance_name']))
print("Private DNS: {}".format(dns_name))
print("Private IP: {}".format(ip_address))
- print("Instance ID: {}".format(dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
+ print("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name'])))
++ notebook_config['instance_name'])))
print("Instance type: {}".format(notebook_config['instance_type']))
print("Key name: {}".format(notebook_config['key_name']))
print("User key name: {}".format(notebook_config['user_keyname']))
@@@ -344,8 -344,8 +343,8 @@@
with open("/root/result.json", 'w') as result:
res = {"hostname": dns_name,
"ip": ip_address,
- "instance_id": dlab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
+ "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'],
- notebook_config['instance_name']),
++ notebook_config['instance_name']),
"master_keyname": os.environ['conf_key_name'],
"notebook_name": notebook_config['instance_name'],
"notebook_image_name": notebook_config['notebook_image_name'],
diff --cc infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
index bc452fc,a2089b2..ca1e9d8
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_dataengine-service_create_configs.py
@@@ -21,20 -21,20 +21,12 @@@
#
# ******************************************************************************
--import boto3
--from botocore.client import Config
--from fabric.api import *
import argparse
--import os
--import sys
--import time
--from fabric.api import lcd
--from fabric.contrib.files import exists
--from fabvenv import virtualenv
- from dlab.notebook_lib import *
- from dlab.actions_lib import *
- from dlab.fab import *
- from dlab.common_lib import *
-from datalab.notebook_lib import *
+ from datalab.actions_lib import *
-from datalab.fab import *
+ from datalab.common_lib import *
++from datalab.fab import *
++from datalab.notebook_lib import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--bucket', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
index 7a04296,56def14..f6b09b9
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_install_dataengine-service_kernels.py
@@@ -22,10 -22,10 +22,9 @@@
# ******************************************************************************
import argparse
--from fabric.api import *
--import boto3
- from dlab.meta_lib import *
-from datalab.meta_lib import *
import os
++from datalab.meta_lib import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--bucket', type=str, default='')
@@@ -55,15 -55,15 +54,16 @@@ def configure_notebook(args)
put(templates_dir + 'dataengine-service_interpreter_spark.json', '/tmp/dataengine-service_interpreter.json')
put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application),
'/tmp/zeppelin_dataengine-service_create_configs.py')
-- sudo('\cp /tmp/zeppelin_dataengine-service_create_configs.py /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
++ sudo(
++ '\cp /tmp/zeppelin_dataengine-service_create_configs.py /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
sudo('chmod 755 /usr/local/bin/zeppelin_dataengine-service_create_configs.py')
- sudo('mkdir -p /usr/lib/python2.7/dlab/')
- run('mkdir -p /tmp/dlab_libs/')
- local('scp -i {} /usr/lib/python2.7/dlab/* {}:/tmp/dlab_libs/'.format(args.keyfile, env.host_string))
- run('chmod a+x /tmp/dlab_libs/*')
- sudo('mv /tmp/dlab_libs/* /usr/lib/python2.7/dlab/')
+ sudo('mkdir -p /usr/lib/python2.7/datalab/')
+ run('mkdir -p /tmp/datalab_libs/')
+ local('scp -i {} /usr/lib/python2.7/datalab/* {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
+ run('chmod a+x /tmp/datalab_libs/*')
+ sudo('mv /tmp/datalab_libs/* /usr/lib/python2.7/datalab/')
if exists('/usr/lib64'):
- sudo('ln -fs /usr/lib/python2.7/dlab /usr/lib64/python2.7/dlab')
+ sudo('ln -fs /usr/lib/python2.7/datalab /usr/lib64/python2.7/datalab')
if __name__ == "__main__":
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
index 29660b0,9be90bd..10d3dfa
--- a/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_collect_data.py
@@@ -22,23 -22,23 +22,20 @@@
# ******************************************************************************
import argparse
++import ast
import json
--import datetime
--from fabric.api import *
- from dlab.actions_lib import *
- from dlab.meta_lib import *
- from dlab.fab import *
- import traceback
+import sys
- import ast
-
++import traceback
+ from datalab.actions_lib import *
-from datalab.meta_lib import *
+ from datalab.fab import *
-import traceback
-import sys
-import ast
-
++from datalab.meta_lib import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--resource_group_name', type=str, default='')
parser.add_argument('--list_resources', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
data = ast.literal_eval(args.list_resources.replace('\'', '"'))
statuses = {}
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_create_datalake_directory.py
index 0493b07,ffab1f5..55c110b
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_datalake_directory.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_datalake_directory.py
@@@ -22,11 -22,11 +22,10 @@@
# ******************************************************************************
import argparse
--import json
- from dlab.fab import *
- from dlab.actions_lib import *
- from dlab.meta_lib import *
-from datalab.fab import *
+import sys
+ from datalab.actions_lib import *
++from datalab.fab import *
+ from datalab.meta_lib import *
-import sys
parser = argparse.ArgumentParser()
parser.add_argument('--resource_group_name', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_create_instance.py
index 295e191,d98da78..9ae95a6
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_instance.py
@@@ -23,10 -23,10 +23,9 @@@
import argparse
import json
- from dlab.actions_lib import *
- from dlab.meta_lib import *
+import sys
-
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
-
parser = argparse.ArgumentParser()
parser.add_argument('--instance_name', type=str, default='')
@@@ -89,9 -89,9 +88,10 @@@ if __name__ == "__main__"
disk_id = disk.id
print("Creating instance {}".format(args.instance_name))
AzureActions().create_instance(args.region, args.instance_size, args.service_base_name,
- args.instance_name, args.dlab_ssh_user_name, args.public_key,
+ args.instance_name, args.datalab_ssh_user_name, args.public_key,
network_interface_id, args.resource_group_name, args.primary_disk_size,
-- args.instance_type, args.image_name, json.loads(args.tags), args.project_name,
++ args.instance_type, args.image_name, json.loads(args.tags),
++ args.project_name,
create_option, disk_id, args.instance_storage_account_type,
args.image_type)
except Exception as err:
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
index 5746fb8,b1b41d2..487c896
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_notebook_image.py
@@@ -21,20 -21,20 +21,16 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
++import json
import os
--import traceback
import sys
--import json
from fabric.api import *
--
if __name__ == "__main__":
try:
- AzureMeta = dlab.meta_lib.AzureMeta()
- AzureActions = dlab.actions_lib.AzureActions()
+ AzureMeta = datalab.meta_lib.AzureMeta()
+ AzureActions = datalab.actions_lib.AzureActions()
image_conf = dict()
image_conf['service_base_name'] = os.environ['conf_service_base_name']
image_conf['resource_group_name'] = os.environ['azure_resource_group_name']
@@@ -78,7 -78,7 +74,8 @@@
image = AzureMeta.get_image(image_conf['resource_group_name'], image_conf['full_image_name'])
if image == '':
print('Creating image from existing notebook.')
- dlab.actions_lib.prepare_vm_for_image(True, image_conf['dlab_ssh_user'], instance_hostname, keyfile_name)
- datalab.actions_lib.prepare_vm_for_image(True, image_conf['datalab_ssh_user'], instance_hostname, keyfile_name)
++ datalab.actions_lib.prepare_vm_for_image(True, image_conf['datalab_ssh_user'], instance_hostname,
++ keyfile_name)
AzureActions.create_image_from_instance(image_conf['resource_group_name'],
image_conf['instance_name'],
os.environ['azure_region'],
@@@ -94,9 -94,9 +91,10 @@@
instance_running = True
instance_hostname = AzureMeta.get_private_ip_address(image_conf['resource_group_name'],
image_conf['instance_name'])
- dlab.actions_lib.remount_azure_disk(True, image_conf['dlab_ssh_user'], instance_hostname, keyfile_name)
- dlab.fab.set_git_proxy(image_conf['dlab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_hostname))
- datalab.actions_lib.remount_azure_disk(True, image_conf['datalab_ssh_user'], instance_hostname, keyfile_name)
++ datalab.actions_lib.remount_azure_disk(True, image_conf['datalab_ssh_user'], instance_hostname,
++ keyfile_name)
+ datalab.fab.set_git_proxy(image_conf['datalab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_hostname))
++ 'http://{}:3128'.format(edge_instance_hostname))
additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
.format(instance_hostname, image_conf['instance_name'], keyfile_name,
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
index 62cd0d7,0f9526c..6e7e026
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_security_group.py
@@@ -21,11 -21,11 +21,11 @@@
#
# ******************************************************************************
--import json
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
++import json
+import sys
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
parser = argparse.ArgumentParser()
parser.add_argument('--resource_group_name', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
index d2e3dcc,75d8432..922b993
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_storage_account.py
@@@ -23,11 -23,11 +23,10 @@@
import argparse
import json
- from dlab.fab import *
- from dlab.actions_lib import *
- from dlab.meta_lib import *
-from datalab.fab import *
+import sys
-
+ from datalab.actions_lib import *
++from datalab.fab import *
+ from datalab.meta_lib import *
-import sys
-
parser = argparse.ArgumentParser()
parser.add_argument('--container_name', type=str, default='')
@@@ -36,7 -36,7 +35,6 @@@ parser.add_argument('--resource_group_n
parser.add_argument('--region', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
try:
check_account = False
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_create_subnet.py
index 6e57137,ced1b9a..3bc237f
--- a/infrastructure-provisioning/src/general/scripts/azure/common_create_subnet.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_create_subnet.py
@@@ -22,12 -22,12 +22,10 @@@
# ******************************************************************************
import argparse
--import json
- from dlab.actions_lib import *
- from dlab.meta_lib import *
- import sys
+import ipaddress
-
++import sys
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys
-import ipaddress
-
parser = argparse.ArgumentParser()
parser.add_argument('--resource_group_name', type=str, default='')
@@@ -69,7 -69,7 +67,8 @@@ if __name__ == "__main__"
print("REQUESTED SUBNET {} ALREADY EXISTS".format(args.subnet_name))
else:
print("Creating Subnet {}".format(args.subnet_name))
- AzureActions().create_subnet(args.resource_group_name, args.vpc_name, args.subnet_name, dlab_subnet_cidr)
- AzureActions().create_subnet(args.resource_group_name, args.vpc_name, args.subnet_name, datalab_subnet_cidr)
++ AzureActions().create_subnet(args.resource_group_name, args.vpc_name, args.subnet_name,
++ datalab_subnet_cidr)
else:
print("Subnet name can't be empty")
sys.exit(1)
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
index 2a9e606,2bd8d82..a291925
--- a/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_download_git_certfile.py
@@@ -22,11 -22,11 +22,10 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
- from fabric.api import *
+import os
-
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
+ from fabric.api import *
-import os
-
parser = argparse.ArgumentParser()
parser.add_argument('--keyfile', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
index 2e697eb,2e418de..23aa32a
--- a/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_notebook_configure_dataengine.py
@@@ -21,16 -21,16 +21,13 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
+import json
- import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
--import uuid
--from fabric.api import *
++import sys
import traceback
++from fabric.api import *
def clear_resources():
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
index a4dda9d,7da5264..f3bb393
--- a/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_prepare_notebook.py
@@@ -21,18 -21,18 +21,15 @@@
#
# ******************************************************************************
--import logging
++import datalab.meta_lib
import json
--import sys
++import logging
import os
--from Crypto.PublicKey import RSA
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
++import sys
import traceback
++from Crypto.PublicKey import RSA
from fabric.api import *
--
if __name__ == "__main__":
instance_class = 'notebook'
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -53,7 -53,7 +50,7 @@@
notebook_config['endpoint_name'] = os.environ['endpoint_name']
notebook_config['endpoint_tag'] = notebook_config['endpoint_name']
notebook_config['application'] = os.environ['application'].lower()
--
++
print('Generating infrastructure names and tags')
try:
notebook_config['exploratory_name'] = os.environ['exploratory_name']
@@@ -138,12 -138,12 +135,13 @@@
print('ERROR: Edge node is unavailable! Aborting...')
ssn_hostname = AzureMeta.get_private_ip_address(notebook_config['resource_group_name'],
os.environ['conf_service_base_name'] + '-ssn')
- dlab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_dlab_path'], os.environ['conf_os_user'],
- datalab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_datalab_path'], os.environ['conf_os_user'],
-- ssn_hostname)
- dlab.fab.append_result("Edge node is unavailable")
++ datalab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_datalab_path'],
++ os.environ['conf_os_user'],
++ ssn_hostname)
+ datalab.fab.append_result("Edge node is unavailable")
sys.exit(1)
except Exception as err:
- dlab.fab.append_result("Failed to verify edge status.", str(err))
+ datalab.fab.append_result("Failed to verify edge status.", str(err))
sys.exit(1)
with open('/root/result.json', 'w') as f:
@@@ -177,5 -177,5 +175,5 @@@
AzureActions.remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
except:
print("The instance hasn't been created.")
- dlab.fab.append_result("Failed to create instance.", str(err))
- sys.exit(1)
+ datalab.fab.append_result("Failed to create instance.", str(err))
- sys.exit(1)
++ sys.exit(1)
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_remove_remote_kernels.py
index 830969b,c9fca0c..51f900c
--- a/infrastructure-provisioning/src/general/scripts/azure/common_remove_remote_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_remove_remote_kernels.py
@@@ -21,12 -21,12 +21,11 @@@
#
# ******************************************************************************
--import os
--import sys
import argparse
-from fabric.api import *
-from datalab.fab import find_cluster_kernels
++import sys
+ from datalab.actions_lib import *
++from datalab.fab import find_cluster_kernels
+from fabric.api import *
- from dlab.fab import find_cluster_kernels
- from dlab.actions_lib import *
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
index f4d7cb4,81f272e..fa4a5d5
--- a/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_reupload_key.py
@@@ -23,11 -23,11 +23,10 @@@
import argparse
-from fabric.api import *
+ from datalab.actions_lib import *
-from datalab.meta_lib import *
+ from datalab.fab import *
-import json
++from datalab.meta_lib import *
+from fabric.api import *
- from dlab.actions_lib import *
- from dlab.meta_lib import *
- from dlab.fab import *
- import json
parser = argparse.ArgumentParser()
parser.add_argument('--conf_resource', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
index ab3c080,794fc6b..522e409
--- a/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_start_notebook.py
@@@ -21,19 -21,19 +21,14 @@@
#
# ******************************************************************************
--import logging
++import datalab.meta_lib
import json
++import logging
++import os
import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
import traceback
--import os
--import uuid
--import argparse
from fabric.api import *
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id'])
@@@ -132,7 -132,7 +127,5 @@@
print(json.dumps(res))
result.write(json.dumps(res))
except Exception as err:
- dlab.fab.append_result("Error with writing results", str(err))
+ datalab.fab.append_result("Error with writing results", str(err))
sys.exit(1)
--
--
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
index 5e77666,7b89c04..8344590
--- a/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_stop_notebook.py
@@@ -21,15 -21,15 +21,12 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+import json
- import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
--import uuid
--import argparse
import sys
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
index 73eab17,7af97b3..072132b
--- a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
@@@ -21,14 -21,14 +21,13 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+import json
- import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
--import uuid
++import sys
import traceback
diff --cc infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
index c74abfe,5a3f300..37e091a
--- a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook_image.py
@@@ -21,13 -21,13 +21,10 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import sys
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import sys
import json
import os
--
++import sys
if __name__ == "__main__":
try:
diff --cc infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index 8d90b5e,54dd46f..c30e00e
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@@ -21,19 -21,19 +21,17 @@@
#
# ******************************************************************************
-import json
-import time
-from fabric.api import *
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
-import traceback
-import sys
-import os
-import uuid
+import json
- import time
- from fabric.api import *
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import traceback
- import sys
- import os
- import uuid
import logging
--from Crypto.PublicKey import RSA
import multiprocessing
++import os
++import sys
++import traceback
++from Crypto.PublicKey import RSA
++from fabric.api import *
def configure_slave(slave_number, data_engine):
@@@ -62,7 -62,7 +60,8 @@@
additional_config = {"user_keyname": data_engine['project_name'],
"user_keydir": os.environ['conf_key_dir']}
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
- slave_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem", json.dumps(additional_config), data_engine['dlab_ssh_user'])
- slave_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem", json.dumps(additional_config), data_engine['datalab_ssh_user'])
++ slave_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem",
++ json.dumps(additional_config), data_engine['datalab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('install_user_key', params))
except:
@@@ -230,11 -230,11 +229,11 @@@ if __name__ == "__main__"
sys.exit(1)
try:
- logging.info('[CREATING DLAB SSH USER ON MASTER NODE]')
- print('[CREATING DLAB SSH USER ON MASTER NODE]')
- params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
+ logging.info('[CREATING DATA ATA LAB SSH USER ON MASTER NODE]')
+ print('[CREATING DATALAB SSH USER ON MASTER NODE]')
- params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
++ params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format \
(master_node_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem", initial_user,
- data_engine['dlab_ssh_user'], sudo_group)
+ data_engine['datalab_ssh_user'], sudo_group)
try:
local("~/scripts/{}.py {}".format('create_ssh_user', params))
@@@ -300,8 -300,8 +299,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES ON MASTER NODE]')
print('[INSTALLING PREREQUISITES ON MASTER NODE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(master_node_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(master_node_hostname, keyfile_name, data_engine['datalab_ssh_user'], data_engine['region'],
edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
diff --cc infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
index 86dc7a9,db321f3..e752d92
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_prepare.py
@@@ -21,20 -21,20 +21,14 @@@
#
# ******************************************************************************
-import json
-import time
-from fabric.api import *
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import traceback
-import sys
-import os
-import uuid
+import json
- import time
- from fabric.api import *
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import traceback
- import sys
- import os
- import uuid
import logging
++import os
++import sys
++import traceback
from Crypto.PublicKey import RSA
--import multiprocessing
--
++from fabric.api import *
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -145,12 -145,12 +139,13 @@@
print('ERROR: Edge node is unavailable! Aborting...')
ssn_hostname = AzureMeta.get_private_ip_address(data_engine['resource_group_name'],
data_engine['service_base_name'] + '-ssn')
- dlab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_dlab_path'], os.environ['conf_os_user'],
- datalab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_datalab_path'], os.environ['conf_os_user'],
-- ssn_hostname)
- dlab.fab.append_result("Edge node is unavailable")
++ datalab.fab.put_resource_status('edge', 'Unavailable', os.environ['ssn_datalab_path'],
++ os.environ['conf_os_user'],
++ ssn_hostname)
+ datalab.fab.append_result("Edge node is unavailable")
sys.exit(1)
except Exception as err:
- dlab.fab.append_result("Failed to verify edge status.", str(err))
+ datalab.fab.append_result("Failed to verify edge status.", str(err))
sys.exit(1)
if os.environ['conf_os_family'] == 'debian':
diff --cc infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
index 308912f,96f2874..04b2009
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_start.py
@@@ -21,15 -21,15 +21,14 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+import json
- import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
++import sys
import traceback
--import uuid
from fabric.api import *
diff --cc infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
index 963c555,8871efd..d03dd52
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_stop.py
@@@ -21,15 -21,15 +21,14 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+import json
- import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
++import sys
import traceback
--import uuid
def stop_data_engine(resource_group_name, cluster_name):
diff --cc infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
index 1363eb8,cc40fef..b78b63f
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_terminate.py
@@@ -21,15 -21,15 +21,14 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+import json
- import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
++import sys
import traceback
--import uuid
def terminate_data_engine(resource_group_name, notebook_name, os_user, key_path, cluster_name):
diff --cc infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
index 9c8f24c,d2d5c8c..6fd43a0
--- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
@@@ -21,17 -21,17 +21,14 @@@
#
# ******************************************************************************
--import logging
++import datalab.meta_lib
import json
++import logging
++import os
import sys
import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
--import os
from fabric.api import *
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id'])
@@@ -210,15 -210,15 +207,15 @@@
"--scala_version {4} --spark_version {5} " \
"--hadoop_version {6} --region {7} " \
"--r_mirror {8} --ip_address {9} --exploratory_name {10} --edge_ip {11}" \
- .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'],
- .format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'],
-- os.environ['notebook_jupyter_version'], os.environ['notebook_scala_version'],
-- os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'],
-- os.environ['azure_region'], os.environ['notebook_r_mirror'],
-- notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
++ .format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'],
++ os.environ['notebook_jupyter_version'], os.environ['notebook_scala_version'],
++ os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'],
++ os.environ['azure_region'], os.environ['notebook_r_mirror'],
++ notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_deep_learning_node', params))
- dlab.actions_lib.remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
++ os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
except:
traceback.print_exc()
raise Exception
@@@ -265,8 -265,8 +262,8 @@@
image = AzureMeta.get_image(notebook_config['resource_group_name'], notebook_config['expected_image_name'])
if image == '':
print("Looks like it's first time we configure notebook server. Creating image.")
- dlab.actions_lib.prepare_vm_for_image(True, notebook_config['dlab_ssh_user'], instance_hostname,
- keyfile_name)
+ datalab.actions_lib.prepare_vm_for_image(True, notebook_config['datalab_ssh_user'], instance_hostname,
- keyfile_name)
++ keyfile_name)
AzureActions.create_image_from_instance(notebook_config['resource_group_name'],
notebook_config['instance_name'],
os.environ['azure_region'],
@@@ -281,10 -281,10 +278,10 @@@
instance_running = True
instance_hostname = AzureMeta.get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
- dlab.actions_lib.remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
- keyfile_name)
- dlab.fab.set_git_proxy(notebook_config['dlab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_private_hostname))
+ datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- keyfile_name)
++ keyfile_name)
+ datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_private_hostname))
++ 'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
.format(instance_hostname, notebook_config['instance_name'], keyfile_name,
diff --cc infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
index f331c01,92aee35..1c0c522
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
@@@ -21,14 -21,14 +21,11 @@@
#
# ******************************************************************************
-import json
-import sys
-import time
-import os
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
+import json
- import sys
- import time
- import os
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
import logging
++import os
++import sys
import traceback
import uuid
from fabric.api import *
@@@ -232,9 -232,9 +229,9 @@@ if __name__ == "__main__"
edge_conf['keycloak_client_secret'] = str(uuid.uuid4())
params = "--hostname {} --keyfile {} --user {} --keycloak_client_id {} --keycloak_client_secret {} " \
"--step_cert_sans '{}'".format(
- edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'],
- edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['datalab_ssh_user'],
-- edge_conf['service_base_name'] + '-' + edge_conf['project_name'] + '-' + edge_conf['endpoint_name'],
-- edge_conf['keycloak_client_secret'], edge_conf['step_cert_sans'])
++ edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['datalab_ssh_user'],
++ edge_conf['service_base_name'] + '-' + edge_conf['project_name'] + '-' + edge_conf['endpoint_name'],
++ edge_conf['keycloak_client_secret'], edge_conf['step_cert_sans'])
try:
local("~/scripts/{}.py {}".format('configure_nginx_reverse_proxy', params))
diff --cc infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
index 9dc1b01,ef6c41c..fafcbb1
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
@@@ -22,16 -22,16 +22,17 @@@
# ******************************************************************************
import json
- from dlab.fab import *
- from dlab.meta_lib import *
-from datalab.fab import *
-from datalab.meta_lib import *
--import sys, time, os
- from dlab.actions_lib import *
-from datalab.actions_lib import *
++import os
++import sys
import traceback
from Crypto.PublicKey import RSA
--
++from datalab.actions_lib import *
++from datalab.fab import *
++from datalab.meta_lib import *
if __name__ == "__main__":
-- local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
++ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
++ os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
diff --cc infrastructure-provisioning/src/general/scripts/azure/edge_start.py
index 04e57ae,5fc42be..cfcf39b
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_start.py
@@@ -21,14 -21,14 +21,11 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import os
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
-import os
++import json
import logging
++import os
import sys
--import json
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -79,6 -79,6 +76,5 @@@
print(json.dumps(res))
result.write(json.dumps(res))
except Exception as err:
- dlab.fab.append_result("Error with writing results", str(err))
+ datalab.fab.append_result("Error with writing results", str(err))
sys.exit(1)
--
diff --cc infrastructure-provisioning/src/general/scripts/azure/edge_status.py
index 1b3fd15,0ab99bc..bb3aa67
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_status.py
@@@ -22,17 -22,17 +22,12 @@@
# ******************************************************************************
--import json
--import sys
--import time
--import os
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
import logging
--from fabric.api import *
++import os
++import sys
import traceback
--
++from fabric.api import *
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
diff --cc infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
index dfc4cba,dc9c042..37c9720
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_stop.py
@@@ -9,9 -9,9 +9,9 @@@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
--#
++#
# http://www.apache.org/licenses/LICENSE-2.0
--#
++#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@@ -20,14 -20,14 +20,11 @@@
# under the License.
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
++import json
import logging
import os
import sys
--import json
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -63,6 -63,6 +60,5 @@@
print(json.dumps(res))
result.write(json.dumps(res))
except Exception as err:
- dlab.fab.append_result("Error with writing results", str(err))
+ datalab.fab.append_result("Error with writing results", str(err))
sys.exit(1)
--
diff --cc infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
index a61c75d,16c3fa2..a10b1e8
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_terminate.py
@@@ -21,14 -21,14 +21,13 @@@
#
# ******************************************************************************
-import json
-import sys
-import time
-import os
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+import json
- import sys
- import time
- import os
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
import logging
++import os
++import sys
import traceback
diff --cc infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index 49f9872,27866a6..42b4a80
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@@ -21,17 -21,17 +21,14 @@@
#
# ******************************************************************************
--import logging
++import datalab.meta_lib
import json
++import logging
++import os
import sys
import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
--import os
from fabric.api import *
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id'])
@@@ -171,8 -171,8 +168,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['azure_region'],
edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -200,8 -200,8 +197,8 @@@
notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_jupyter_node', params))
- dlab.actions_lib.remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
++ os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
except:
traceback.print_exc()
raise Exception
@@@ -267,8 -267,8 +264,8 @@@
notebook_config['expected_image_name'])
if image == '':
print("Looks like it's first time we configure notebook server. Creating image.")
- dlab.actions_lib.prepare_vm_for_image(True, notebook_config['dlab_ssh_user'], instance_hostname,
- keyfile_name)
+ datalab.actions_lib.prepare_vm_for_image(True, notebook_config['datalab_ssh_user'], instance_hostname,
- keyfile_name)
++ keyfile_name)
AzureActions.create_image_from_instance(notebook_config['resource_group_name'],
notebook_config['instance_name'],
os.environ['azure_region'],
@@@ -283,10 -283,10 +280,10 @@@
instance_running = True
instance_hostname = AzureMeta.get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
- dlab.actions_lib.remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
- keyfile_name)
- dlab.fab.set_git_proxy(notebook_config['dlab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_private_hostname))
+ datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- keyfile_name)
++ keyfile_name)
+ datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_private_hostname))
++ 'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
.format(instance_hostname, notebook_config['instance_name'], keyfile_name,
diff --cc infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
index 4c44bbd,aa8864d..92fc876
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyterlab_configure.py
@@@ -21,16 -21,16 +21,13 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
+import json
- import sys
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
--from fabric.api import *
++import sys
import traceback
--
++from fabric.api import *
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -133,11 -133,11 +130,11 @@@
sys.exit(1)
try:
- logging.info('[CREATING DLAB SSH USER]')
- print('[CREATING DLAB SSH USER]')
- params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
+ logging.info('[CREATING DATALAB SSH USER]')
+ print('[CREATING DATALAB SSH USER]')
- params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
++ params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format \
(instance_hostname, os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem", initial_user,
- notebook_config['dlab_ssh_user'], sudo_group)
+ notebook_config['datalab_ssh_user'], sudo_group)
try:
local("~/scripts/{}.py {}".format('create_ssh_user', params))
@@@ -171,8 -171,8 +168,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['azure_region'],
edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -200,8 -200,8 +197,8 @@@
notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_jupyterlab_node', params))
- dlab.actions_lib.remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
++ os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
except:
traceback.print_exc()
raise Exception
@@@ -248,13 -248,13 +245,13 @@@
image = AzureMeta.get_image(notebook_config['resource_group_name'], notebook_config['expected_image_name'])
if image == '':
print("Looks like it's first time we configure notebook server. Creating image.")
- dlab.actions_lib.prepare_vm_for_image(True, notebook_config['dlab_ssh_user'], instance_hostname,
- keyfile_name)
+ datalab.actions_lib.prepare_vm_for_image(True, notebook_config['datalab_ssh_user'], instance_hostname,
- keyfile_name)
++ keyfile_name)
AzureActions.create_image_from_instance(notebook_config['resource_group_name'],
-- notebook_config['instance_name'],
-- os.environ['azure_region'],
-- notebook_config['expected_image_name'],
-- json.dumps(notebook_config['image_tags']))
++ notebook_config['instance_name'],
++ os.environ['azure_region'],
++ notebook_config['expected_image_name'],
++ json.dumps(notebook_config['image_tags']))
print("Image was successfully created.")
local("~/scripts/{}.py".format('common_prepare_notebook'))
instance_running = False
@@@ -264,10 -264,10 +261,10 @@@
instance_running = True
instance_hostname = AzureMeta.get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
- dlab.actions_lib.remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
- keyfile_name)
- dlab.fab.set_git_proxy(notebook_config['dlab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_private_hostname))
+ datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- keyfile_name)
++ keyfile_name)
+ datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_private_hostname))
++ 'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
.format(instance_hostname, notebook_config['instance_name'], keyfile_name,
diff --cc infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
index e2d481d,a09c554..92db0a2
--- a/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_prepare.py
@@@ -21,20 -21,20 +21,15 @@@
#
# ******************************************************************************
++import datalab.meta_lib
import json
--import sys
--import time
++import logging
import os
--import re
++import sys
import traceback
from Crypto.PublicKey import RSA
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
--import logging
from fabric.api import *
--
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id'])
diff --cc infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
index a78480c,952fc2e..7549eac
--- a/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/project_terminate.py
@@@ -21,15 -21,15 +21,14 @@@
#
# ******************************************************************************
++import datalab.actions_lib
++import datalab.fab
++import datalab.meta_lib
import json
import logging
--import sys
--import time
import os
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
import requests
++import sys
import traceback
diff --cc infrastructure-provisioning/src/general/scripts/azure/rstudio_change_pass.py
index cbc97c4,645c323..6af1829
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_change_pass.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_change_pass.py
@@@ -21,11 -21,11 +21,10 @@@
#
# ******************************************************************************
--from fabric.api import *
import argparse
- from dlab.fab import *
-from datalab.fab import *
import sys
--
++from datalab.fab import *
++from fabric.api import *
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
@@@ -34,7 -34,7 +33,6 @@@ parser.add_argument('--os_user', type=s
parser.add_argument('--rstudio_pass', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
print("Configure connections")
env['connection_attempts'] = 100
diff --cc infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
index 8487238,b96f495..91a5945
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
@@@ -21,17 -21,17 +21,14 @@@
#
# ******************************************************************************
--import logging
++import datalab.meta_lib
import json
++import logging
++import os
import sys
import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
-import datalab.fab
-import datalab.actions_lib
-import datalab.meta_lib
--import os
from fabric.api import *
--
if __name__ == "__main__":
instance_class = 'notebook'
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -172,8 -172,8 +169,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES TO R_STUDIO NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO R_STUDIO NOTEBOOK INSTANCE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['azure_region'],
edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -200,8 -200,8 +197,8 @@@
notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_rstudio_node', params))
- dlab.actions_lib.remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
+ datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
++ os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem")
except:
traceback.print_exc()
raise Exception
@@@ -266,8 -266,8 +263,8 @@@
notebook_config['expected_image_name'])
if image == '':
print("Looks like it's first time we configure notebook server. Creating image.")
- dlab.actions_lib.prepare_vm_for_image(True, notebook_config['dlab_ssh_user'], instance_hostname,
- keyfile_name)
+ datalab.actions_lib.prepare_vm_for_image(True, notebook_config['datalab_ssh_user'], instance_hostname,
- keyfile_name)
++ keyfile_name)
AzureActions.create_image_from_instance(notebook_config['resource_group_name'],
notebook_config['instance_name'],
os.environ['azure_region'],
@@@ -282,10 -282,10 +279,10 @@@
instance_running = True
instance_hostname = AzureMeta.get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
- dlab.actions_lib.remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
- keyfile_name)
- dlab.fab.set_git_proxy(notebook_config['dlab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_private_hostname))
+ datalab.actions_lib.remount_azure_disk(True, notebook_config['datalab_ssh_user'], instance_hostname,
- keyfile_name)
++ keyfile_name)
+ datalab.fab.set_git_proxy(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name,
- 'http://{}:3128'.format(edge_instance_private_hostname))
++ 'http://{}:3128'.format(edge_instance_private_hostname))
additional_config = {"proxy_host": edge_instance_private_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
.format(instance_hostname, notebook_config['instance_name'], keyfile_name,
@@@ -361,13 -361,13 +358,13 @@@
{"description": "RStudio",
"url": rstudio_notebook_access_url},
{"description": "Ungit",
-- "url": rstudio_ungit_access_url}#,
-- #{"description": "RStudio (via tunnel)",
++ "url": rstudio_ungit_access_url} # ,
++ # {"description": "RStudio (via tunnel)",
# "url": rstudio_ip_url},
-- #{"description": "Ungit (via tunnel)",
++ # {"description": "Ungit (via tunnel)",
# "url": ungit_ip_url}
],
- "exploratory_user": notebook_config['dlab_ssh_user'],
+ "exploratory_user": notebook_config['datalab_ssh_user'],
"exploratory_pass": notebook_config['rstudio_pass']}
result.write(json.dumps(res))
except Exception as err:
diff --cc infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
index fa3f827,db84948..6fad0bb
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
@@@ -21,16 -21,16 +21,13 @@@
#
# ******************************************************************************
--import sys
--import os
-import json
-from fabric.api import *
-import datalab.ssn_lib
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
+import json
- from fabric.api import *
- import dlab.ssn_lib
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
import logging
++import os
++import sys
import traceback
++from fabric.api import *
if __name__ == "__main__":
local_log_filename = "{}_{}.log".format(os.environ['conf_resource'], os.environ['request_id'])
@@@ -56,11 -56,11 +53,11 @@@
try:
- AzureMeta = dlab.meta_lib.AzureMeta()
- AzureActions = dlab.actions_lib.AzureActions()
+ AzureMeta = datalab.meta_lib.AzureMeta()
+ AzureActions = datalab.actions_lib.AzureActions()
ssn_conf = dict()
ssn_conf['instance'] = 'ssn'
--
++
logging.info('[DERIVING NAMES]')
print('[DERIVING NAMES]')
@@@ -133,10 -133,10 +130,11 @@@
sys.exit(1)
try:
- logging.info('[CREATING DLAB SSH USER]')
- print('[CREATING DLAB SSH USER]')
- params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
- (ssn_conf['instance_host'], ssn_conf['ssh_key_path'], ssn_conf['initial_user'], ssn_conf['dlab_ssh_user'],
+ logging.info('[CREATING DATALAB SSH USER]')
+ print('[CREATING DATALAB SSH USER]')
- params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format\
- (ssn_conf['instance_host'], ssn_conf['ssh_key_path'], ssn_conf['initial_user'], ssn_conf['datalab_ssh_user'],
++ params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format \
++ (ssn_conf['instance_host'], ssn_conf['ssh_key_path'], ssn_conf['initial_user'],
++ ssn_conf['datalab_ssh_user'],
ssn_conf['sudo_group'])
local("~/scripts/{}.py {}".format('create_ssh_user', params))
except Exception as err:
@@@ -502,8 -502,8 +500,10 @@@
os.environ['conf_resource'], ssn_conf['service_base_name'], os.environ['conf_cloud_provider'],
ssn_conf['billing_enabled'], ssn_conf['azure_auth_path'], os.environ['azure_offer_number'],
os.environ['azure_currency'], os.environ['azure_locale'], os.environ['azure_region_info'],
-- ssn_conf['ldap_login'], ssn_conf['tenant_id'], ssn_conf['datalake_application_id'], ssn_conf['datalake_store_name'], json.dumps(cloud_params),
-- ssn_conf['subscription_id'], os.environ['azure_validate_permission_scope'], ssn_conf['default_endpoint_name'],
++ ssn_conf['ldap_login'], ssn_conf['tenant_id'], ssn_conf['datalake_application_id'],
++ ssn_conf['datalake_store_name'], json.dumps(cloud_params),
++ ssn_conf['subscription_id'], os.environ['azure_validate_permission_scope'],
++ ssn_conf['default_endpoint_name'],
os.environ['keycloak_client_name'], os.environ['keycloak_client_secret'],
os.environ['keycloak_auth_server_url'])
local("~/scripts/{}.py {}".format('configure_ui', params))
@@@ -579,9 -579,9 +579,9 @@@
f.write(json.dumps(res))
print('Upload response file')
-- params = "--instance_name {} --local_log_filepath {} --os_user {} --instance_hostname {}".\
- format(ssn_conf['instance_name'], local_log_filepath, ssn_conf['dlab_ssh_user'], ssn_conf['instnace_ip'])
++ params = "--instance_name {} --local_log_filepath {} --os_user {} --instance_hostname {}". \
+ format(ssn_conf['instance_name'], local_log_filepath, ssn_conf['datalab_ssh_user'], ssn_conf['instnace_ip'])
local("~/scripts/{}.py {}".format('upload_response_file', params))
except Exception as err:
- dlab.fab.append_result("Error with writing results.", str(err))
+ datalab.fab.append_result("Error with writing results.", str(err))
sys.exit(1)
diff --cc infrastructure-provisioning/src/general/scripts/azure/ssn_create_datalake.py
index 53a94e9,b9884de..6f9ffc7
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_datalake.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_datalake.py
@@@ -23,11 -23,11 +23,10 @@@
import argparse
import json
- from dlab.fab import *
- from dlab.actions_lib import *
- from dlab.meta_lib import *
-from datalab.fab import *
+import sys
-
+ from datalab.actions_lib import *
++from datalab.fab import *
+ from datalab.meta_lib import *
-import sys
-
parser = argparse.ArgumentParser()
parser.add_argument('--datalake_name', type=str, default='')
@@@ -36,7 -36,7 +35,6 @@@ parser.add_argument('--resource_group_n
parser.add_argument('--region', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
try:
check_datalake = False
diff --cc infrastructure-provisioning/src/general/scripts/azure/ssn_create_peering.py
index 6b53174,0093f27..4f2d342
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_peering.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_peering.py
@@@ -22,9 -22,9 +22,10 @@@
# ******************************************************************************
import argparse
- from dlab.actions_lib import *
- from dlab.meta_lib import *
- import sys, time
++import sys
++import time
+ from datalab.actions_lib import *
+ from datalab.meta_lib import *
-import sys, time
parser = argparse.ArgumentParser()
parser.add_argument('--source_resource_group_name', type=str, default='')
diff --cc infrastructure-provisioning/src/general/scripts/azure/ssn_create_resource_group.py
index 8134096,dc296eb..c6f761c
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_resource_group.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_resource_group.py
@@@ -30,7 -30,7 +30,6 @@@ parser.add_argument('--resource_group_n
parser.add_argument('--region', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
if args.resource_group_name != '':
if AzureMeta().get_resource_group(args.resource_group_name):
diff --cc infrastructure-provisioning/src/general/scripts/azure/ssn_create_vpc.py
index ac7cb18,eeea9ec..faeefbd
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_create_vpc.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_create_vpc.py
@@@ -32,7 -32,7 +32,6 @@@ parser.add_argument('--region', type=st
parser.add_argument('--vpc_cidr', type=str, default='')
args = parser.parse_args()
--
if __name__ == "__main__":
if args.vpc_name != '':
if AzureMeta().get_vpc(args.resource_group_name, args.vpc_name):
diff --cc infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
index 408f423,93d3dd8..b6b8a89
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_prepare.py
@@@ -21,22 -21,22 +21,18 @@@
#
# ******************************************************************************
--import sys
--import os
-import json
-from fabric.api import *
-from Crypto.PublicKey import RSA
-import datalab.ssn_lib
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
+import json
- from fabric.api import *
- from Crypto.PublicKey import RSA
- import dlab.ssn_lib
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
import logging
++import os
++import sys
import traceback
--
++from Crypto.PublicKey import RSA
++from fabric.api import *
if __name__ == "__main__":
local_log_filename = "{}_{}.log".format(os.environ['conf_resource'], os.environ['request_id'])
-- local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
++ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
filename=local_log_filepath)
diff --cc infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
index d1bf732,45aee06..7c7399a
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_terminate.py
@@@ -21,16 -21,16 +21,16 @@@
#
# ******************************************************************************
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
- import dlab.ssn_lib
- import sys
- import os
- from fabric.api import *
-import datalab.fab
+ import datalab.actions_lib
++import datalab.fab
+ import datalab.meta_lib
+ import datalab.ssn_lib
-import sys
-import os
-from fabric.api import *
++import json
import logging
++import os
++import sys
import traceback
--import json
++from fabric.api import *
def terminate_ssn_node(resource_group_name, service_base_name, vpc_name, region):
@@@ -140,14 -140,14 +140,14 @@@ if __name__ == "__main__"
level=logging.DEBUG,
filename=local_log_filepath)
# generating variables dictionary
- AzureMeta = dlab.meta_lib.AzureMeta()
- AzureActions = dlab.actions_lib.AzureActions()
+ AzureMeta = datalab.meta_lib.AzureMeta()
+ AzureActions = datalab.actions_lib.AzureActions()
print('Generating infrastructure names and tags')
ssn_conf = dict()
- ssn_conf['service_base_name'] = dlab.fab.replace_multi_symbols(os.environ['conf_service_base_name'][:20],
- '-', True)
+ ssn_conf['service_base_name'] = datalab.fab.replace_multi_symbols(os.environ['conf_service_base_name'][:20],
- '-', True)
++ '-', True)
ssn_conf['resource_group_name'] = os.environ.get(
-- 'azure_resource_group_name', '{}-resource-group'.format(ssn_conf['service_base_name']))
++ 'azure_resource_group_name', '{}-resource-group'.format(ssn_conf['service_base_name']))
ssn_conf['region'] = os.environ['azure_region']
ssn_conf['vpc_name'] = os.environ['azure_vpc_name']
diff --cc infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
index 914f686,926b381..f23795a
--- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
@@@ -21,18 -21,18 +21,14 @@@
#
# ******************************************************************************
--import logging
-import json
-import sys
-import traceback
-import datalab.fab
-import datalab.actions_lib
+ import datalab.meta_lib
+import json
- import sys
- import traceback
- import dlab.fab
- import dlab.actions_lib
- import dlab.meta_lib
++import logging
import os
++import sys
import traceback
from fabric.api import *
--
if __name__ == "__main__":
instance_class = 'notebook'
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@@ -174,8 -174,8 +170,8 @@@
try:
logging.info('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
-- params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
++ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], os.environ['azure_region'],
edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@@ -199,8 -199,8 +195,8 @@@
notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_tensor_node', params))
... 19205 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org