You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/04/16 12:55:26 UTC
[incubator-dlab] branch DLAB-569 updated: [DLAB-569]: Fixed issue
with NTP in private subnet
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-569
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-569 by this push:
new 9d0fc97 [DLAB-569]: Fixed issue with NTP in private subnet
9d0fc97 is described below
commit 9d0fc97b38ff91c4fb86064b3e7ccd78638a6e2c
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Tue Apr 16 15:55:17 2019 +0300
[DLAB-569]: Fixed issue with NTP in private subnet
---
.../src/base/scripts/install_prerequisites.py | 3 ++-
.../src/general/lib/os/debian/common_lib.py | 7 ++++--
.../src/general/lib/os/redhat/common_lib.py | 6 +++++-
.../general/scripts/aws/dataengine_configure.py | 11 ++++++----
.../general/scripts/aws/deeplearning_configure.py | 7 +++---
.../src/general/scripts/aws/jupyter_configure.py | 9 +++++---
.../src/general/scripts/aws/rstudio_configure.py | 6 ++++--
.../scripts/aws/tensor-rstudio_configure.py | 9 +++++---
.../src/general/scripts/aws/tensor_configure.py | 9 +++++---
.../src/general/scripts/aws/zeppelin_configure.py | 6 ++++--
.../general/scripts/azure/dataengine_configure.py | 25 +++++++++++++---------
.../scripts/azure/deeplearning_configure.py | 6 +++---
.../src/general/scripts/azure/jupyter_configure.py | 5 +++--
.../src/general/scripts/azure/rstudio_configure.py | 5 +++--
.../src/general/scripts/azure/tensor_configure.py | 5 +++--
.../general/scripts/azure/zeppelin_configure.py | 5 +++--
.../general/scripts/gcp/dataengine_configure.py | 20 +++++++++++------
.../src/general/scripts/gcp/jupyter_configure.py | 6 ++++--
.../src/general/scripts/gcp/rstudio_configure.py | 9 +++++---
.../scripts/gcp/tensor-rstudio_configure.py | 9 +++++---
.../src/general/scripts/gcp/tensor_configure.py | 12 +++++++----
.../src/general/scripts/gcp/zeppelin_configure.py | 13 +++++------
22 files changed, 123 insertions(+), 70 deletions(-)
diff --git a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
index 7d694f0..0170b45 100644
--- a/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
+++ b/infrastructure-provisioning/src/base/scripts/install_prerequisites.py
@@ -37,6 +37,7 @@ parser.add_argument('--keyfile', type=str, default='')
parser.add_argument('--pip_packages', type=str, default='boto3 argparse fabric==1.14.0 awscli google-api-python-client google-auth-httplib2 google-cloud-storage pycrypto azure==2.0.0')
parser.add_argument('--additional_config', type=str, default='{"empty":"string"}')
parser.add_argument('--user', type=str, default='')
+parser.add_argument('--edge_private_ip', type=str, default='')
parser.add_argument('--region', type=str, default='')
args = parser.parse_args()
@@ -72,6 +73,6 @@ if __name__ == "__main__":
ensure_pip(args.pip_packages)
print("Installing NTPd")
- ensure_ntpd(args.user)
+ ensure_ntpd(args.user, args.edge_ip)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
index 94f12fe..2633b76 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
@@ -74,11 +74,14 @@ def find_java_path_local():
return java_path
-def ensure_ntpd(user):
+def ensure_ntpd(user, edge_ip=''):
try:
if not exists('/home/{}/.ensure_dir/ntpd_ensured'.format(user)):
sudo('timedatectl set-ntp no')
- sudo('apt-get -y install ntp')
+ sudo('apt-get -y install ntp ntpdate')
+ if os.environ['conf_resource'] != 'ssn' or os.environ['conf_resource'] != 'edge':
+ sudo('echo "server {} prefer iburst" >> /etc/ntp.conf'.format(edge_ip))
+ sudo('systemctl restart ntp')
sudo('systemctl enable ntp')
sudo('touch /home/{}/.ensure_dir/ntpd_ensured'.format(user))
except:
diff --git a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
index c93c1ee..190ba35 100644
--- a/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/redhat/common_lib.py
@@ -24,6 +24,7 @@
from fabric.api import *
from fabric.contrib.files import exists
import sys
+import os
def ensure_pkg(user, requisites='git vim gcc python-devel openssl-devel nmap libffi libffi-devel unzip'):
@@ -75,11 +76,14 @@ def find_java_path_local():
return java_path
-def ensure_ntpd(user):
+def ensure_ntpd(user, edge_ip=''):
try:
if not exists('/home/{}/.ensure_dir/ntpd_ensured'.format(user)):
sudo('yum -y install ntp')
sudo('systemctl start ntpd')
+ if os.environ['conf_resource'] != 'ssn' or os.environ['conf_resource'] != 'edge':
+ sudo('echo "server {} prefer iburst" >> /etc/ntp.conf'.format(edge_ip))
+ sudo('systemctl restart ntp')
sudo('systemctl enable ntpd')
sudo('touch /home/{}/.ensure_dir/ntpd_ensured'.format(user))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
index 57e66fd..96fb8d2 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine_configure.py
@@ -102,8 +102,9 @@ def configure_slave(slave_number, data_engine):
try:
logging.info('[INSTALLING PREREQUISITES ON SLAVE NODE]')
print('[INSTALLING PREREQUISITES ON SLAVE NODE]')
- params = "--hostname {} --keyfile {} --user {} --region {}". \
- format(slave_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(slave_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
@@ -189,6 +190,7 @@ if __name__ == "__main__":
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
edge_instance_name = os.environ['conf_service_base_name'] + "-" + os.environ['edge_user_name'] + '-edge'
edge_instance_hostname = get_instance_hostname(data_engine['tag_name'], edge_instance_name)
+ edge_instance_private_ip = get_instance_ip_address(data_engine['tag_name'], edge_instance_name).get('Private')
if data_engine['network_type'] == 'private':
edge_instance_ip = get_instance_ip_address(data_engine['tag_name'], edge_instance_name).get('Private')
else:
@@ -279,8 +281,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES ON MASTER NODE]')
print('[INSTALLING PREREQUISITES ON MASTER NODE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(master_node_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(master_node_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
index 50b31a1..069ebd9 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
@@ -71,6 +71,7 @@ if __name__ == "__main__":
instance_hostname = get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
edge_instance_name = os.environ['conf_service_base_name'] + "-" + os.environ['edge_user_name'] + '-edge'
edge_instance_hostname = get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
if notebook_config['network_type'] == 'private':
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
else:
@@ -142,9 +143,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO DEEPLEARNING NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO DEEPLEARNING NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".format(instance_hostname, keyfile_name,
- notebook_config['dlab_ssh_user'],
- os.environ['aws_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".format(
+ instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index 3a352cc..7f32c24 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@ -71,6 +71,7 @@ if __name__ == "__main__":
instance_hostname = get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
edge_instance_name = os.environ['conf_service_base_name'] + "-" + os.environ['edge_user_name'] + '-edge'
edge_instance_hostname = get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
if notebook_config['network_type'] == 'private':
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
else:
@@ -108,7 +109,8 @@ if __name__ == "__main__":
print('[CONFIGURE PROXY ON JUPYTER INSTANCE]')
additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}"\
- .format(instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ .format(instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config),
+ notebook_config['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('common_configure_proxy', params))
except:
@@ -124,8 +126,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
index 506b3b8..c35380a 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
@@ -73,6 +73,7 @@ if __name__ == "__main__":
instance_hostname = get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
edge_instance_name = os.environ['conf_service_base_name'] + "-" + os.environ['edge_user_name'] + '-edge'
edge_instance_hostname = get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
if notebook_config['network_type'] == 'private':
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
else:
@@ -127,8 +128,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO R_STUDIO NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO R_STUDIO NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
index eb7a3b7..a32be53 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
@@ -77,6 +77,7 @@ if __name__ == "__main__":
instance_hostname = get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
edge_instance_name = os.environ['conf_service_base_name'] + "-" + os.environ['edge_user_name'] + '-edge'
edge_instance_hostname = get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
if notebook_config['network_type'] == 'private':
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
else:
@@ -114,7 +115,8 @@ if __name__ == "__main__":
print('[CONFIGURE PROXY ON TENSORFLOW-RSTUDIO INSTANCE]')
additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}"\
- .format(instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ .format(instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config),
+ notebook_config['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('common_configure_proxy', params))
except:
@@ -130,8 +132,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO TENSORFLOW-RSTUDIO NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO TENSORFLOW-RSTUDIO NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
index 1ac5305..046aab7 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
@@ -76,6 +76,7 @@ if __name__ == "__main__":
instance_hostname = get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
edge_instance_name = os.environ['conf_service_base_name'] + "-" + os.environ['edge_user_name'] + '-edge'
edge_instance_hostname = get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
if notebook_config['network_type'] == 'private':
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
else:
@@ -113,7 +114,8 @@ if __name__ == "__main__":
print('[CONFIGURE PROXY ON TENSOR INSTANCE]')
additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}"\
- .format(instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ .format(instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config),
+ notebook_config['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('common_configure_proxy', params))
except:
@@ -129,8 +131,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
index a0c2b49..c17ed5b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
@@ -81,6 +81,7 @@ if __name__ == "__main__":
instance_hostname = get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name'])
edge_instance_name = os.environ['conf_service_base_name'] + "-" + os.environ['edge_user_name'] + '-edge'
edge_instance_hostname = get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
+ edge_instance_private_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
if notebook_config['network_type'] == 'private':
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
else:
@@ -135,8 +136,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO ZEPPELIN NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO ZEPPELIN NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}" \
- .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}" \
+ .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
index f93702f..f897481 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine_configure.py
@@ -124,8 +124,9 @@ def configure_slave(slave_number, data_engine):
try:
logging.info('[INSTALLING PREREQUISITES ON SLAVE NODE]')
print('[INSTALLING PREREQUISITES ON SLAVE NODE]')
- params = "--hostname {} --keyfile {} --user {} --region {}". \
- format(slave_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(slave_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'],
+ edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
@@ -144,7 +145,8 @@ def configure_slave(slave_number, data_engine):
try:
logging.info('[CONFIGURE SLAVE NODE {}]'.format(slave + 1))
print('[CONFIGURE SLAVE NODE {}]'.format(slave + 1))
- params = "--hostname {} --keyfile {} --region {} --spark_version {} --hadoop_version {} --os_user {} --scala_version {} --r_mirror {} --master_ip {} --node_type {}". \
+ params = "--hostname {} --keyfile {} --region {} --spark_version {} --hadoop_version {} --os_user {} " \
+ "--scala_version {} --r_mirror {} --master_ip {} --node_type {}". \
format(slave_hostname, keyfile_name, data_engine['region'], os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], data_engine['dlab_ssh_user'],
os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'], master_node_hostname,
@@ -195,8 +197,8 @@ if __name__ == "__main__":
data_engine['private_subnet_cidr'] = AzureMeta().get_subnet(data_engine['resource_group_name'],
data_engine['vpc_name'],
data_engine['private_subnet_name']).address_prefix
- data_engine['master_security_group_name'] = '{}-{}-dataengine-master-sg'.format(data_engine['service_base_name'],
- data_engine['user_name'])
+ data_engine['master_security_group_name'] = '{}-{}-dataengine-master-sg'.format(
+ data_engine['service_base_name'], data_engine['user_name'])
data_engine['slave_security_group_name'] = '{}-{}-dataengine-slave-sg'.format(data_engine['service_base_name'],
data_engine['user_name'])
data_engine['cluster_name'] = '{}-{}-de-{}-{}'.format(data_engine['service_base_name'],
@@ -215,7 +217,7 @@ if __name__ == "__main__":
data_engine['master_node_name'])
edge_instance_name = '{}-{}-edge'.format(data_engine['service_base_name'], data_engine['user_name'])
edge_instance_private_hostname = AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
- edge_instance_name)
+ edge_instance_name)
if os.environ['conf_network_type'] == 'private':
edge_instance_hostname = AzureMeta().get_private_ip_address(data_engine['resource_group_name'],
edge_instance_name)
@@ -268,7 +270,8 @@ if __name__ == "__main__":
additional_config = {"user_keyname": os.environ['edge_user_name'],
"user_keydir": os.environ['conf_key_dir']}
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
- master_node_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem", json.dumps(additional_config), data_engine['dlab_ssh_user'])
+ master_node_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem", json.dumps(
+ additional_config), data_engine['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('install_user_key', params))
except:
@@ -327,8 +330,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES ON MASTER NODE]')
print('[INSTALLING PREREQUISITES ON MASTER NODE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(master_node_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(master_node_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'],
+ edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
@@ -347,7 +351,8 @@ if __name__ == "__main__":
try:
logging.info('[CONFIGURE MASTER NODE]')
print('[CONFIGURE MASTER NODE]')
- params = "--hostname {} --keyfile {} --region {} --spark_version {} --hadoop_version {} --os_user {} --scala_version {} --r_mirror {} --master_ip {} --node_type {}".\
+ params = "--hostname {} --keyfile {} --region {} --spark_version {} --hadoop_version {} --os_user {} " \
+ "--scala_version {} --r_mirror {} --master_ip {} --node_type {}".\
format(master_node_hostname, keyfile_name, data_engine['region'], os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], data_engine['dlab_ssh_user'],
os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'], master_node_hostname,
diff --git a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
index 2d8e133..38c2fb1 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
@@ -150,9 +150,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO DEEPLEARNING NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO DEEPLEARNING NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".format(instance_hostname, keyfile_name,
- notebook_config['dlab_ssh_user'],
- os.environ['azure_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".format(
+ instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
+ edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index dc57bec..f4aeea6 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -132,8 +132,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
+ edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
index c85b0b7..d75455a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
@@ -134,8 +134,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO R_STUDIO NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO R_STUDIO NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
+ edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
index 28c2048..704f372 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
@@ -133,8 +133,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
+ edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
index 505e930..367e9f7 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
@@ -134,8 +134,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO ZEPPELIN NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO ZEPPELIN NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}" \
- .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}" \
+ .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['azure_region'],
+ edge_instance_private_hostname)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
index 9104bde..ec55d66 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_configure.py
@@ -65,7 +65,8 @@ def configure_slave(slave_number, data_engine):
additional_config = {"user_keyname": os.environ['edge_user_name'],
"user_keydir": os.environ['conf_key_dir']}
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
- slave_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem", json.dumps(additional_config), data_engine['dlab_ssh_user'])
+ slave_hostname, os.environ['conf_key_dir'] + data_engine['key_name'] + ".pem", json.dumps(
+ additional_config), data_engine['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('install_user_key', params))
except:
@@ -104,8 +105,9 @@ def configure_slave(slave_number, data_engine):
try:
logging.info('[INSTALLING PREREQUISITES ON SLAVE NODE]')
print('[INSTALLING PREREQUISITES ON SLAVE NODE]')
- params = "--hostname {} --keyfile {} --user {} --region {}". \
- format(slave_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(slave_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
@@ -124,7 +126,8 @@ def configure_slave(slave_number, data_engine):
try:
logging.info('[CONFIGURE SLAVE NODE {}]'.format(slave + 1))
print('[CONFIGURE SLAVE NODE {}]'.format(slave + 1))
- params = "--hostname {} --keyfile {} --region {} --spark_version {} --hadoop_version {} --os_user {} --scala_version {} --r_mirror {} --master_ip {} --node_type {}". \
+ params = "--hostname {} --keyfile {} --region {} --spark_version {} --hadoop_version {} --os_user {} " \
+ "--scala_version {} --r_mirror {} --master_ip {} --node_type {}". \
format(slave_hostname, keyfile_name, data_engine['region'], os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], data_engine['dlab_ssh_user'],
os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'], master_node_hostname,
@@ -207,6 +210,7 @@ if __name__ == "__main__":
edge_instance_name = '{0}-{1}-edge'.format(data_engine['service_base_name'],
data_engine['edge_user_name'])
edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name)
data_engine['dlab_ssh_user'] = os.environ['conf_os_user']
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
except Exception as err:
@@ -285,8 +289,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES ON MASTER NODE]')
print('[INSTALLING PREREQUISITES ON MASTER NODE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(master_node_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(master_node_hostname, keyfile_name, data_engine['dlab_ssh_user'], data_engine['region'],
+ edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
@@ -305,7 +310,8 @@ if __name__ == "__main__":
try:
logging.info('[CONFIGURE MASTER NODE]')
print('[CONFIGURE MASTER NODE]')
- params = "--hostname {} --keyfile {} --region {} --spark_version {} --hadoop_version {} --os_user {} --scala_version {} --r_mirror {} --master_ip {} --node_type {}".\
+ params = "--hostname {} --keyfile {} --region {} --spark_version {} --hadoop_version {} --os_user {} " \
+ "--scala_version {} --r_mirror {} --master_ip {} --node_type {}".\
format(master_node_hostname, keyfile_name, data_engine['region'], os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], data_engine['dlab_ssh_user'],
os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'], master_node_hostname,
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index 3b8956f..ad4f8f1 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -57,6 +57,7 @@ if __name__ == "__main__":
instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
edge_instance_name = '{0}-{1}-edge'.format(notebook_config['service_base_name'], notebook_config['edge_user_name'])
edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name)
notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
@@ -109,8 +110,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'], os.environ['gcp_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".\
+ format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'],
+ os.environ['gcp_region'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
index a9910c7..1c23f38 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
@@ -59,6 +59,7 @@ if __name__ == "__main__":
instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
edge_instance_name = '{0}-{1}-edge'.format(notebook_config['service_base_name'], notebook_config['edge_user_name'])
edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name)
notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
@@ -95,7 +96,8 @@ if __name__ == "__main__":
print('[CONFIGURE PROXY ON RSTUDIO INSTANCE]')
additional_config = {"proxy_host": edge_instance_name, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
- .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'], json.dumps(additional_config),
+ .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'],
+ json.dumps(additional_config),
notebook_config['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('common_configure_proxy', params))
@@ -112,8 +114,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO RSTUDIO NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO RSTUDIO NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}". \
- format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'], os.environ['gcp_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'],
+ os.environ['gcp_region'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
index 14dfd32..0955584 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
@@ -65,6 +65,7 @@ if __name__ == "__main__":
instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
edge_instance_name = '{0}-{1}-edge'.format(notebook_config['service_base_name'], notebook_config['edge_user_name'])
edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name)
notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
@@ -101,7 +102,8 @@ if __name__ == "__main__":
print('[CONFIGURE PROXY ON TENSORFLOW-RSTUDIO INSTANCE]')
additional_config = {"proxy_host": edge_instance_name, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
- .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'], json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'],
+ json.dumps(additional_config), notebook_config['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('common_configure_proxy', params))
except:
@@ -117,8 +119,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO TENSORFLOW-RSTUDIO NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO TENSORFLOW-RSTUDIO NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}". \
- format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'], os.environ['gcp_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'],
+ os.environ['gcp_region'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
index 847135d..c646f14 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
@@ -60,6 +60,7 @@ if __name__ == "__main__":
instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
edge_instance_name = '{0}-{1}-edge'.format(notebook_config['service_base_name'], notebook_config['edge_user_name'])
edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name)
notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
@@ -112,8 +113,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO TENSOR NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}". \
- format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'], os.environ['gcp_region'])
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
+ format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'],
+ os.environ['gcp_region'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
@@ -150,7 +152,8 @@ if __name__ == "__main__":
additional_config = {"user_keyname": os.environ['edge_user_name'],
"user_keydir": os.environ['conf_key_dir']}
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
- instance_hostname, notebook_config['ssh_key_path'], json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ instance_hostname, notebook_config['ssh_key_path'], json.dumps(additional_config),
+ notebook_config['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('install_user_key', params))
except:
@@ -191,7 +194,8 @@ if __name__ == "__main__":
if image_id_list and image_id_list[0] != '':
print("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
- print("Looks like another image creating operation for your template have been started a moment ago.")
+ print("Looks like another image creating operation for your template have been started a "
+ "moment ago.")
if image_id_list and image_id_list[1] != '':
print("Image of secondary disk was successfully created. It's ID is {}".format(image_id_list[1]))
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
index eaaaba1..0830ff0 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
@@ -60,6 +60,7 @@ if __name__ == "__main__":
instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
edge_instance_name = '{0}-{1}-edge'.format(notebook_config['service_base_name'], notebook_config['edge_user_name'])
edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name)
notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
@@ -112,9 +113,9 @@ if __name__ == "__main__":
try:
logging.info('[INSTALLING PREREQUISITES TO ZEPPELIN NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO ZEPPELIN NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --user {} --region {}". \
+ params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}". \
format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'],
- os.environ['gcp_region'])
+ os.environ['gcp_region'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
except:
@@ -143,10 +144,10 @@ if __name__ == "__main__":
"--livy_version {} --multiple_clusters {} " \
"--r_mirror {} --endpoint_url {} " \
"--exploratory_name {}" \
- .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'], os.environ['gcp_region'],
- json.dumps(additional_config), notebook_config['dlab_ssh_user'], os.environ['notebook_spark_version'],
- os.environ['notebook_hadoop_version'], edge_instance_name, '3128',
- os.environ['notebook_zeppelin_version'], os.environ['notebook_scala_version'],
+ .format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'],
+ os.environ['gcp_region'], json.dumps(additional_config), notebook_config['dlab_ssh_user'],
+ os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'], edge_instance_name,
+ '3128', os.environ['notebook_zeppelin_version'], os.environ['notebook_scala_version'],
os.environ['notebook_livy_version'], os.environ['notebook_multiple_clusters'],
os.environ['notebook_r_mirror'], 'null',
notebook_config['exploratory_name'])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org