You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/03/21 11:30:35 UTC
[incubator-dlab] 01/01: added reverse proxy to azure
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-252
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 9c6f4977bc208505cbc53a1d0d70bc71a2985905
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Thu Mar 21 13:30:26 2019 +0200
added reverse proxy to azure
---
.../src/general/files/azure/base_Dockerfile | 1 +
.../general/scripts/azure/dataengine_configure.py | 53 ++++++++++++++++-
.../scripts/azure/deeplearning_configure.py | 66 +++++++++++++++++----
.../src/general/scripts/azure/edge_configure.py | 67 ++++++++++++++++++----
.../src/general/scripts/azure/edge_prepare.py | 22 +++++++
.../src/general/scripts/azure/jupyter_configure.py | 44 +++++++++++++-
.../src/general/scripts/azure/rstudio_configure.py | 44 +++++++++++++-
.../src/general/scripts/azure/tensor_configure.py | 52 +++++++++++++++--
.../general/scripts/azure/zeppelin_configure.py | 44 +++++++++++++-
9 files changed, 358 insertions(+), 35 deletions(-)
diff --git a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
index e3c47b2..aeb7b62 100644
--- a/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/azure/base_Dockerfile
@@ -68,6 +68,7 @@ COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python2.7/dlab/commo
COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python2.7/dlab/fab.py
COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
+COPY ${SRC_PATH}edge/templates/locations/ /root/locations/
RUN chmod a+x /root/*.py && \
chmod a+x /root/scripts/* && \
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index b0ae221..064bed1 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@ -210,6 +210,7 @@ if __name__ == "__main__":
data_engine['instance_count'] = int(os.environ['dataengine_instance_count'])
data_engine['slave_size'] = os.environ['azure_dataengine_slave_size']
data_engine['dlab_ssh_user'] = os.environ['conf_os_user']
+ data_engine['notebook_name'] = os.environ['notebook_instance_name']
master_node_hostname = AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
data_engine['master_node_name'])
edge_instance_name = '{}-{}-edge'.format(data_engine['service_base_name'], data_engine['user_name'])
@@ -378,8 +379,51 @@ if __name__ == "__main__":
AzureActions().remove_instance(data_engine['resource_group_name'], data_engine['master_node_name'])
sys.exit(1)
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ notebook_instance_ip = AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
+ data_engine['notebook_name'])
+ additional_info = {
+ "computational_name": data_engine['computational_name'],
+ "master_node_hostname": master_node_hostname,
+ "notebook_instance_ip": notebook_instance_ip,
+ "instance_count": data_engine['instance_count'],
+ "master_node_name": data_engine['master_node_name'],
+ "slave_node_name": data_engine['slave_node_name'],
+ "tensor": False
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ keyfile_name,
+ data_engine['dlab_ssh_user'],
+ 'spark',
+ data_engine['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ for i in range(data_engine['instance_count'] - 1):
+ slave_name = data_engine['slave_node_name'] + '{}'.format(i + 1)
+ AzureActions().remove_instance(data_engine['resource_group_name'], slave_name)
+ AzureActions().remove_instance(data_engine['resource_group_name'], data_engine['master_node_name'])
+ sys.exit(1)
try:
+ ip_address = AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
+ data_engine['master_node_name'])
+ spark_master_url = "http://" + ip_address + ":8080"
+ spark_master_acces_url = "http://" + edge_instance_hostname + "/{}/".format(
+ data_engine['exploratory_name'] + '_' + data_engine['computational_name'])
logging.info('[SUMMARY]')
print('[SUMMARY]')
print("Service base name: {}".format(data_engine['service_base_name']))
@@ -392,7 +436,14 @@ if __name__ == "__main__":
res = {"hostname": data_engine['cluster_name'],
"instance_id": data_engine['master_node_name'],
"key_name": data_engine['key_name'],
- "Action": "Create new Data Engine"}
+ "Action": "Create new Data Engine",
+ "computational_url": [
+ {"description": "Apache Spark Master",
+ "url": spark_master_acces_url},
+ # {"description": "Apache Spark Master (via tunnel)",
+ # "url": spark_master_url}
+ ]
+ }
print(json.dumps(res))
result.write(json.dumps(res))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
index a34b8dd..ab14a5b 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
@@ -252,12 +252,49 @@ if __name__ == "__main__":
append_result("Failed creating image.", str(err))
AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
sys.exit(1)
+
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ additional_info = {
+ 'instance_hostname': instance_hostname,
+ 'tensor': True
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ keyfile_name,
+ notebook_config['dlab_ssh_user'],
+ 'jupyter',
+ notebook_config['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to set edge reverse proxy template.", str(err))
+ AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
try:
ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
- tensor_board_url = 'http://' + ip_address + ':6006'
- jupyter_url = 'http://' + ip_address + ':8888/{}/'.format(notebook_config['exploratory_name'])
+ tensorboard_ip_url = 'http://' + ip_address + ':6006'
+ jupyter_ip_url = 'http://' + ip_address + ':8888/{}/'.format(notebook_config['exploratory_name'])
ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name'])
+ jupyter_notebook_acces_url = "http://" + edge_instance_hostname + "/{}/".format(
+ notebook_config['exploratory_name'])
+ jupyter_ungit_acces_url = "http://" + edge_instance_hostname + "/{}-ungit/".format(
+ notebook_config['exploratory_name'])
+ tensorboard_acces_url = "http://" + edge_instance_hostname + "/{}-tensor/".format(
+ notebook_config['exploratory_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['instance_name']))
@@ -269,9 +306,8 @@ if __name__ == "__main__":
print("Jupyter URL: {}".format(jupyter_url))
print("Tensor Board URL: {}".format(tensor_board_url))
print("Ungit URL: {}".format(ungit_ip_url))
- print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format(notebook_config['key_name'],
- notebook_config['dlab_ssh_user'],
- ip_address))
+ print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format(
+ notebook_config['key_name'], notebook_config['dlab_ssh_user'], ip_address))
with open("/root/result.json", 'w') as result:
res = {"ip": ip_address,
@@ -281,13 +317,19 @@ if __name__ == "__main__":
"notebook_image_name": notebook_config['notebook_image_name'],
"Action": "Create new notebook server",
"exploratory_url": [
- {"description": "TensorBoard",
- "url": tensor_board_url},
- {"description": "Jupyter",
- "url": jupyter_url},
- {"description": "Ungit",
- "url": ungit_ip_url}
- ]}
+ {"description": "Jupyter",
+ "url": jupyter_notebook_acces_url},
+ {"description": "TensorBoard",
+ "url": tensorboard_acces_url},
+ {"description": "Ungit",
+ "url": jupyter_ungit_acces_url},
+ {"description": "Jupyter (via tunnel)",
+ "url": jupyter_ip_url},
+ {"description": "TensorBoard (via tunnel)",
+ "url": tensorboard_ip_url},
+ {"description": "Ungit (via tunnel)",
+ "url": ungit_ip_url}
+ ]}
result.write(json.dumps(res))
except Exception as err:
print('Error: {0}'.format(err))
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
index bedf3f1..7a8b5aa 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_configure.py
@@ -48,18 +48,23 @@ if __name__ == "__main__":
edge_conf['user_keyname'] = os.environ['edge_user_name']
edge_conf['private_subnet_name'] = edge_conf['service_base_name'] + '-' + edge_conf['user_name'] + '-subnet'
edge_conf['instance_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + '-edge'
- edge_conf['network_interface_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + '-edge-nif'
- edge_conf['static_public_ip_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + '-edge-ip'
+ edge_conf['network_interface_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + \
+ '-edge-nif'
+ edge_conf['static_public_ip_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + \
+ '-edge-ip'
edge_conf['primary_disk_name'] = edge_conf['instance_name'] + '-disk0'
- edge_conf['instance_dns_name'] = 'host-' + edge_conf['instance_name'] + '.' + edge_conf['region'] + '.cloudapp.azure.com'
+ edge_conf['instance_dns_name'] = 'host-' + edge_conf['instance_name'] + '.' + edge_conf['region'] + \
+ '.cloudapp.azure.com'
edge_conf['user_storage_account_name'] = edge_conf['service_base_name'] + '-' + edge_conf[
'user_name'] + '-storage'
- edge_conf['user_container_name'] = (edge_conf['service_base_name'] + '-' + edge_conf['user_name'] + '-container').lower()
+ edge_conf['user_container_name'] = (edge_conf['service_base_name'] + '-' + edge_conf['user_name'] +
+ '-container').lower()
edge_conf['shared_storage_account_name'] = edge_conf['service_base_name'] + '-shared-storage'
edge_conf['shared_container_name'] = (edge_conf['service_base_name'] + '-shared-container').lower()
edge_conf['datalake_store_name'] = edge_conf['service_base_name'] + '-ssn-datalake'
edge_conf['datalake_shared_directory_name'] = edge_conf['service_base_name'] + '-shared-folder'
- edge_conf['datalake_user_directory_name'] = '{0}-{1}-folder'.format(edge_conf['service_base_name'], edge_conf['user_name'])
+ edge_conf['datalake_user_directory_name'] = '{0}-{1}-folder'.format(edge_conf['service_base_name'],
+ edge_conf['user_name'])
edge_conf['edge_security_group_name'] = edge_conf['instance_name'] + '-sg'
edge_conf['notebook_security_group_name'] = edge_conf['service_base_name'] + "-" + edge_conf['user_name'] + \
'-nb-sg'
@@ -69,11 +74,12 @@ if __name__ == "__main__":
+ edge_conf['user_name'] + '-dataengine-slave-sg'
edge_conf['dlab_ssh_user'] = os.environ['conf_os_user']
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], edge_conf['key_name'])
- edge_conf['private_subnet_cidr'] = AzureMeta().get_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'],
+ edge_conf['private_subnet_cidr'] = AzureMeta().get_subnet(edge_conf['resource_group_name'],
+ edge_conf['vpc_name'],
edge_conf['private_subnet_name']).address_prefix
if os.environ['conf_network_type'] == 'private':
edge_conf['edge_private_ip'] = AzureMeta().get_private_ip_address(edge_conf['resource_group_name'],
- edge_conf['instance_name'])
+ edge_conf['instance_name'])
edge_conf['edge_public_ip'] = edge_conf['edge_private_ip']
else:
edge_conf['edge_public_ip'] = AzureMeta().get_instance_public_ip_address(edge_conf['resource_group_name'],
@@ -91,7 +97,8 @@ if __name__ == "__main__":
AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'],
edge_conf['private_subnet_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'],
+ edge_conf['notebook_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['master_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
@@ -131,7 +138,8 @@ if __name__ == "__main__":
AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'],
edge_conf['private_subnet_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'],
+ edge_conf['notebook_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['master_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
@@ -162,7 +170,8 @@ if __name__ == "__main__":
AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'],
edge_conf['private_subnet_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'],
+ edge_conf['notebook_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['master_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
@@ -202,7 +211,8 @@ if __name__ == "__main__":
AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'],
edge_conf['private_subnet_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'],
+ edge_conf['notebook_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['master_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
@@ -236,7 +246,40 @@ if __name__ == "__main__":
AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'],
edge_conf['private_subnet_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name'])
- AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['notebook_security_group_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'],
+ edge_conf['notebook_security_group_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'],
+ edge_conf['master_security_group_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'],
+ edge_conf['slave_security_group_name'])
+ for storage_account in AzureMeta().list_storage_accounts(edge_conf['resource_group_name']):
+ if edge_conf['user_storage_account_name'] == storage_account.tags["Name"]:
+ AzureActions().remove_storage_account(edge_conf['resource_group_name'], storage_account.name)
+ if os.environ['azure_datalake_enable'] == 'true':
+ for datalake in AzureMeta().list_datalakes(edge_conf['resource_group_name']):
+ if edge_conf['datalake_store_name'] == datalake.tags["Name"]:
+ AzureActions().remove_datalake_directory(datalake.name, edge_conf['datalake_user_directory_name'])
+ sys.exit(1)
+
+ try:
+ print('[INSTALLING NGINX REVERSE PROXY]')
+ logging.info('[INSTALLING NGINX REVERSE PROXY]')
+ params = "--hostname {} --keyfile {} --user {}" \
+ .format(instance_hostname, keyfile_name, edge_conf['dlab_ssh_user'])
+ try:
+ local("~/scripts/{}.py {}".format('configure_nginx_reverse_proxy', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed installing Nginx reverse proxy. Excpeption: " + str(err))
+ AzureActions().remove_instance(edge_conf['resource_group_name'], edge_conf['instance_name'])
+ AzureActions().remove_subnet(edge_conf['resource_group_name'], edge_conf['vpc_name'],
+ edge_conf['private_subnet_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'], edge_conf['edge_security_group_name'])
+ AzureActions().remove_security_group(edge_conf['resource_group_name'],
+ edge_conf['notebook_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
edge_conf['master_security_group_name'])
AzureActions().remove_security_group(edge_conf['resource_group_name'],
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
index 0654ac2..1976482 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
@@ -164,6 +164,17 @@ if __name__ == "__main__":
"direction": "Inbound"
},
{
+ "name": "in-4",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "80",
+ "source_address_prefix": "*",
+ "destination_address_prefix": "*",
+ "access": "Allow",
+ "priority": 120,
+ "direction": "Inbound"
+ },
+ {
"name": "out-1",
"protocol": "Tcp",
"source_port_range": "*",
@@ -341,6 +352,17 @@ if __name__ == "__main__":
},
{
"name": "out-17",
+ "protocol": "Tcp",
+ "source_port_range": "*",
+ "destination_port_range": "8042",
+ "source_address_prefix": "*",
+ "destination_address_prefix": edge_conf['private_subnet_cidr'],
+ "access": "Allow",
+ "priority": 190,
+ "direction": "Outbound"
+ },
+ {
+ "name": "out-18",
"protocol": "*",
"source_port_range": "*",
"destination_port_range": "*",
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index 66a979e..065f741 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -253,12 +253,47 @@ if __name__ == "__main__":
append_result("Failed creating image from notebook.", str(err))
AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
sys.exit(1)
+
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ additional_info = {
+ 'instance_hostname': instance_hostname,
+ 'tensor': False
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ keyfile_name,
+ notebook_config['dlab_ssh_user'],
+ 'jupyter',
+ notebook_config['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to set edge reverse proxy template.", str(err))
+ AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
# generating output information
try:
ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name'])
ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name'])
+ jupyter_notebook_acces_url = "http://" + edge_instance_hostname + "/{}/".format(
+ notebook_config['exploratory_name'])
+ jupyter_ungit_acces_url = "http://" + edge_instance_hostname + "/{}-ungit/".format(
+ notebook_config['exploratory_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['instance_name']))
@@ -281,9 +316,14 @@ if __name__ == "__main__":
"notebook_image_name": notebook_config['notebook_image_name'],
"exploratory_url": [
{"description": "Jupyter",
- "url": jupyter_ip_url},
+ "url": jupyter_notebook_acces_url},
{"description": "Ungit",
- "url": ungit_ip_url}]}
+ "url": jupyter_ungit_acces_url},
+ {"description": "Jupyter (via tunnel)",
+ "url": jupyter_ip_url},
+ {"description": "Ungit (via tunnel)",
+ "url": ungit_ip_url}
+ ]}
result.write(json.dumps(res))
except Exception as err:
append_result("Failed to generate output information", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
index 0b0e293..61e1680 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
@@ -257,12 +257,47 @@ if __name__ == "__main__":
append_result("Failed creating image.", str(err))
AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
sys.exit(1)
+
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ additional_info = {
+ 'instance_hostname': instance_hostname,
+ 'tensor': False
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ keyfile_name,
+ notebook_config['dlab_ssh_user'],
+ 'rstudio',
+ notebook_config['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to set edge reverse proxy template.", str(err))
+ AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
try:
# generating output information
ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
rstudio_ip_url = "http://" + ip_address + ":8787/"
ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name'])
+ rstudio_notebook_acces_url = "http://" + edge_instance_hostname + "/{}/".format(
+ notebook_config['exploratory_name'])
+ rstudio_ungit_acces_url = "http://" + edge_instance_hostname + "/{}-ungit/".format(
+ notebook_config['exploratory_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['instance_name']))
@@ -287,9 +322,14 @@ if __name__ == "__main__":
"Action": "Create new notebook server",
"exploratory_url": [
{"description": "RStudio",
- "url": rstudio_ip_url},
+ "url": rstudio_notebook_acces_url},
{"description": "Ungit",
- "url": ungit_ip_url}],
+ "url": rstudio_ungit_acces_url},
+ {"description": "RStudio (via tunnel)",
+ "url": rstudio_ip_url},
+ {"description": "Ungit (via tunnel)",
+ "url": ungit_ip_url}
+ ],
"exploratory_user": notebook_config['dlab_ssh_user'],
"exploratory_pass": notebook_config['rstudio_pass']}
result.write(json.dumps(res))
diff --git a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
index 094abd8..b502766 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
@@ -249,6 +249,37 @@ if __name__ == "__main__":
append_result("Failed creating image.", str(err))
AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
sys.exit(1)
+
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ additional_info = {
+ 'instance_hostname': instance_hostname,
+ 'tensor': True
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ keyfile_name,
+ notebook_config['dlab_ssh_user'],
+ 'jupyter',
+ notebook_config['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to set edge reverse proxy template.", str(err))
+ AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
# generating output information
try:
ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
@@ -256,6 +287,12 @@ if __name__ == "__main__":
tensorboard_url = "http://" + ip_address + ":6006/"
jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name'])
ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name'])
+ jupyter_notebook_acces_url = "http://" + edge_instance_hostname + "/{}/".format(
+ notebook_config['exploratory_name'])
+ tensorboard_acces_url = "http://" + edge_instance_hostname + "/{}-tensor/".format(
+ notebook_config['exploratory_name'])
+ jupyter_ungit_acces_url = "http://" + edge_instance_hostname + "/{}-ungit/".format(
+ notebook_config['exploratory_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['instance_name']))
@@ -280,12 +317,19 @@ if __name__ == "__main__":
"instance_id": notebook_config['instance_name'],
"Action": "Create new notebook server",
"exploratory_url": [
- {"description": "TensorBoard",
- "url": tensorboard_url},
{"description": "Jupyter",
- "url": jupyter_ip_url},
+ "url": jupyter_notebook_acces_url},
+ {"description": "TensorBoard",
+ "url": tensorboard_acces_url},
{"description": "Ungit",
- "url": ungit_ip_url}]}
+ "url": jupyter_ungit_acces_url},
+ {"description": "Jupyter (via tunnel)",
+ "url": jupyter_ip_url},
+ {"description": "TensorBoard (via tunnel)",
+ "url": tensorboard_url},
+ {"description": "Ungit (via tunnel)",
+ "url": ungit_ip_url}
+ ]}
result.write(json.dumps(res))
except Exception as err:
print('Error: {0}'.format(err))
diff --git a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
index 527d4aa..5ca4b77 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
@@ -264,12 +264,47 @@ if __name__ == "__main__":
append_result("Failed creating image.", str(err))
AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
sys.exit(1)
+
+ try:
+ print('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
+ additional_info = {
+ 'instance_hostname': instance_hostname,
+ 'tensor': False
+ }
+ params = "--edge_hostname {} " \
+ "--keyfile {} " \
+ "--os_user {} " \
+ "--type {} " \
+ "--exploratory_name {} " \
+ "--additional_info '{}'"\
+ .format(edge_instance_hostname,
+ keyfile_name,
+ notebook_config['dlab_ssh_user'],
+ 'zeppelin',
+ notebook_config['exploratory_name'],
+ json.dumps(additional_info))
+ try:
+ local("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params))
+ except:
+ append_result("Failed edge reverse proxy template")
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to set edge reverse proxy template.", str(err))
+ AzureActions().remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
+ sys.exit(1)
+
# generating output information
try:
ip_address = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
notebook_config['instance_name'])
zeppelin_ip_url = "http://" + ip_address + ":8080/"
ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name'])
+ zeppelin_notebook_acces_url = "http://" + edge_instance_hostname + "/{}/".format(
+ notebook_config['exploratory_name'])
+ zeppelin_ungit_acces_url = "http://" + edge_instance_hostname + "/{}-ungit/".format(
+ notebook_config['exploratory_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['instance_name']))
@@ -292,9 +327,14 @@ if __name__ == "__main__":
"Action": "Create new notebook server",
"exploratory_url": [
{"description": "Apache Zeppelin",
- "url": zeppelin_ip_url},
+ "url": zeppelin_notebook_acces_url},
{"description": "Ungit",
- "url": ungit_ip_url}]}
+ "url": zeppelin_ungit_acces_url},
+ {"description": "Apache Zeppelin (via tunnel)",
+ "url": zeppelin_ip_url},
+ {"description": "Ungit (via tunnel)",
+ "url": ungit_ip_url}
+ ]}
result.write(json.dumps(res))
except Exception as err:
print('Error: {0}'.format(err))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org