You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by bh...@apache.org on 2019/07/26 10:46:05 UTC
[incubator-dlab] branch v2.1.1 updated (5c925d6 -> 8b72aae)
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a change to branch v2.1.1
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
from 5c925d6 Updated README file
new a71344a [DLAB-947] Configure Notebook to be use 22 port for Git
new 8b72aae [DLAB-947] Bugfix(opened 22 port in outbound SGs)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/deeplearning/scripts/configure_deep_learning_node.py | 3 ++-
infrastructure-provisioning/src/general/lib/os/fab.py | 8 +++++++-
.../src/general/scripts/aws/deeplearning_configure.py | 5 +++--
.../src/general/scripts/aws/jupyter_configure.py | 7 +++++--
.../src/general/scripts/aws/rstudio_configure.py | 5 +++--
.../src/general/scripts/aws/tensor-rstudio_configure.py | 5 +++--
.../src/general/scripts/aws/tensor_configure.py | 5 +++--
.../src/general/scripts/aws/zeppelin_configure.py | 5 +++--
.../src/general/scripts/azure/deeplearning_configure.py | 6 ++++--
.../src/general/scripts/azure/edge_prepare.py | 2 +-
.../src/general/scripts/azure/jupyter_configure.py | 6 ++++--
.../src/general/scripts/azure/rstudio_configure.py | 6 ++++--
.../src/general/scripts/azure/tensor_configure.py | 6 ++++--
.../src/general/scripts/azure/zeppelin_configure.py | 6 ++++--
.../src/general/scripts/gcp/deeplearning_configure.py | 5 +++--
.../src/general/scripts/gcp/jupyter_configure.py | 5 +++--
.../src/general/scripts/gcp/rstudio_configure.py | 4 ++--
.../src/general/scripts/gcp/tensor-rstudio_configure.py | 4 ++--
.../src/general/scripts/gcp/tensor_configure.py | 4 ++--
.../src/general/scripts/gcp/zeppelin_configure.py | 5 +++--
.../src/jupyter/scripts/configure_jupyter_node.py | 3 ++-
.../src/rstudio/scripts/configure_rstudio_node.py | 3 ++-
.../src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py | 3 ++-
.../src/tensor/scripts/configure_tensor_node.py | 3 ++-
.../src/zeppelin/scripts/configure_zeppelin_node.py | 3 ++-
25 files changed, 75 insertions(+), 42 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 02/02: [DLAB-947] Bugfix(opened 22 port in
outbound SGs)
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch v2.1.1
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 8b72aae597d25f5d99cfa5c4bf52d16588d41ae8
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Thu Jul 25 17:01:11 2019 +0300
[DLAB-947] Bugfix(opened 22 port in outbound SGs)
---
infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
index 0177ec3..c70aedd 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/edge_prepare.py
@@ -180,7 +180,7 @@ if __name__ == "__main__":
"source_port_range": "*",
"destination_port_range": "22",
"source_address_prefix": "*",
- "destination_address_prefix": edge_conf['private_subnet_cidr'],
+ "destination_address_prefix": "*",
"access": "Allow",
"priority": 100,
"direction": "Outbound"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/02: [DLAB-947] Configure Notebook to be use 22
port for Git
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch v2.1.1
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit a71344a08b4341c9a4cb5ce1e1e7b9be072abc12
Author: Dyoma33 <de...@gmail.com>
AuthorDate: Thu Jul 25 16:36:17 2019 +0300
[DLAB-947] Configure Notebook to be use 22 port for Git
---
.../src/deeplearning/scripts/configure_deep_learning_node.py | 3 ++-
infrastructure-provisioning/src/general/lib/os/fab.py | 8 +++++++-
.../src/general/scripts/aws/deeplearning_configure.py | 5 +++--
.../src/general/scripts/aws/jupyter_configure.py | 7 +++++--
.../src/general/scripts/aws/rstudio_configure.py | 5 +++--
.../src/general/scripts/aws/tensor-rstudio_configure.py | 5 +++--
.../src/general/scripts/aws/tensor_configure.py | 5 +++--
.../src/general/scripts/aws/zeppelin_configure.py | 5 +++--
.../src/general/scripts/azure/deeplearning_configure.py | 6 ++++--
.../src/general/scripts/azure/jupyter_configure.py | 6 ++++--
.../src/general/scripts/azure/rstudio_configure.py | 6 ++++--
.../src/general/scripts/azure/tensor_configure.py | 6 ++++--
.../src/general/scripts/azure/zeppelin_configure.py | 6 ++++--
.../src/general/scripts/gcp/deeplearning_configure.py | 5 +++--
.../src/general/scripts/gcp/jupyter_configure.py | 5 +++--
.../src/general/scripts/gcp/rstudio_configure.py | 4 ++--
.../src/general/scripts/gcp/tensor-rstudio_configure.py | 4 ++--
.../src/general/scripts/gcp/tensor_configure.py | 4 ++--
.../src/general/scripts/gcp/zeppelin_configure.py | 5 +++--
.../src/jupyter/scripts/configure_jupyter_node.py | 3 ++-
.../src/rstudio/scripts/configure_rstudio_node.py | 3 ++-
.../src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py | 3 ++-
.../src/tensor/scripts/configure_tensor_node.py | 3 ++-
.../src/zeppelin/scripts/configure_zeppelin_node.py | 3 ++-
24 files changed, 74 insertions(+), 41 deletions(-)
diff --git a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
index 098cb33..c37da53 100644
--- a/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
+++ b/infrastructure-provisioning/src/deeplearning/scripts/configure_deep_learning_node.py
@@ -43,6 +43,7 @@ parser.add_argument('--hadoop_version', type=str, default='')
parser.add_argument('--r_mirror', type=str, default='')
parser.add_argument('--ip_adress', type=str, default='')
parser.add_argument('--exploratory_name', type=str, default='')
+parser.add_argument('--edge_ip', type=str, default='')
args = parser.parse_args()
@@ -155,7 +156,7 @@ if __name__ == "__main__":
print("Install nodejs")
install_nodejs(args.os_user)
print("Install Ungit")
- install_ungit(args.os_user, args.exploratory_name)
+ install_ungit(args.os_user, args.exploratory_name, args.edge_ip)
if exists('/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
install_gitlab_cert(args.os_user, gitlab_certfile)
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index ff38fec..8d6c4e2 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -406,7 +406,7 @@ def ensure_toree_local_kernel(os_user, toree_link, scala_kernel_path, files_dir,
sys.exit(1)
-def install_ungit(os_user, notebook_name):
+def install_ungit(os_user, notebook_name, edge_ip):
if not exists('/home/{}/.ensure_dir/ungit_ensured'.format(os_user)):
try:
sudo('npm -g install ungit@{}'.format(os.environ['notebook_ungit_version']))
@@ -428,6 +428,12 @@ def install_ungit(os_user, notebook_name):
run('echo "spark-warehouse/" >> ~/.gitignore')
run('echo "metastore_db/" >> ~/.gitignore')
run('echo "derby.log" >> ~/.gitignore')
+ sudo(
+ 'echo -e "Host git.epam.com\n HostName git.epam.com\n ProxyCommand nc -X connect -x {}:3128 %h %p\n" > /home/{}/.ssh/config'.format(
+ edge_ip, os_user))
+ sudo(
+ 'echo -e "Host github.com\n HostName github.com\n ProxyCommand nc -X connect -x {}:3128 %h %p" >> /home/{}/.ssh/config'.format(
+ edge_ip, os_user))
sudo('systemctl daemon-reload')
sudo('systemctl enable ungit.service')
sudo('systemctl start ungit.service')
diff --git a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
index f98ae4d..8c06f57 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/deeplearning_configure.py
@@ -78,6 +78,7 @@ if __name__ == "__main__":
else:
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Public')
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
try:
if os.environ['conf_os_family'] == 'debian':
@@ -165,12 +166,12 @@ if __name__ == "__main__":
"--os_user {2} --jupyter_version {3} " \
"--scala_version {4} --spark_version {5} " \
"--hadoop_version {6} --region {7} " \
- "--r_mirror {8} --ip_adress {9} --exploratory_name {10}" \
+ "--r_mirror {8} --ip_adress {9} --exploratory_name {10} --edge_ip {11}" \
.format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'],
os.environ['notebook_jupyter_version'], os.environ['notebook_scala_version'],
os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'],
os.environ['aws_region'], os.environ['notebook_r_mirror'],
- notebook_config['ip_address'], notebook_config['exploratory_name'])
+ notebook_config['ip_address'], notebook_config['exploratory_name'], edge_ip)
try:
local("~/scripts/{}.py {}".format('configure_deep_learning_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index 2b390b3..86decc8 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@ -78,6 +78,7 @@ if __name__ == "__main__":
else:
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Public')
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
try:
if os.environ['conf_os_family'] == 'debian':
@@ -154,7 +155,8 @@ if __name__ == "__main__":
"--scala_version {6} " \
"--r_mirror {7} " \
"--ip_adress {8} " \
- "--exploratory_name {9}".\
+ "--exploratory_name {9}" \
+ "--edge_ip {10}".\
format(instance_hostname,
keyfile_name,
os.environ['aws_region'],
@@ -164,7 +166,8 @@ if __name__ == "__main__":
os.environ['notebook_scala_version'],
os.environ['notebook_r_mirror'],
notebook_config['ip_address'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'],
+ edge_ip)
try:
local("~/scripts/{}.py {}".format('configure_jupyter_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
index e0b046c..afa424f 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/rstudio_configure.py
@@ -80,6 +80,7 @@ if __name__ == "__main__":
else:
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Public')
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
try:
if os.environ['conf_os_family'] == 'debian':
@@ -150,12 +151,12 @@ if __name__ == "__main__":
params = "--hostname {0} --keyfile {1} " \
"--region {2} --rstudio_pass {3} " \
"--rstudio_version {4} --os_user {5} " \
- "--r_mirror {6} --ip_adress {7} --exploratory_name {8}" \
+ "--r_mirror {6} --ip_adress {7} --exploratory_name {8} --edge_ip {9}" \
.format(instance_hostname, keyfile_name,
os.environ['aws_region'], notebook_config['rstudio_pass'],
os.environ['notebook_rstudio_version'], notebook_config['dlab_ssh_user'],
os.environ['notebook_r_mirror'], notebook_config['ip_address'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_ip)
try:
local("~/scripts/{}.py {}".format('configure_rstudio_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
index 96b4cc3..afe9d73 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor-rstudio_configure.py
@@ -85,6 +85,7 @@ if __name__ == "__main__":
else:
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Public')
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
try:
if os.environ['conf_os_family'] == 'debian':
@@ -155,12 +156,12 @@ if __name__ == "__main__":
params = "--hostname {0} --keyfile {1} " \
"--region {2} --rstudio_pass {3} " \
"--rstudio_version {4} --os_user {5} " \
- "--r_mirror {6} --ip_adress {7} --exploratory_name {8}" \
+ "--r_mirror {6} --ip_adress {7} --exploratory_name {8} --edge_ip {9}" \
.format(instance_hostname, keyfile_name,
os.environ['aws_region'], notebook_config['rstudio_pass'],
os.environ['notebook_rstudio_version'], notebook_config['dlab_ssh_user'],
os.environ['notebook_r_mirror'], notebook_config['ip_address'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_ip)
try:
local("~/scripts/{}.py {}".format('configure_tensor-rstudio_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
index 1230ce1..d5be4fd 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py
@@ -83,6 +83,7 @@ if __name__ == "__main__":
else:
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Public')
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
try:
if os.environ['conf_os_family'] == 'debian':
@@ -152,10 +153,10 @@ if __name__ == "__main__":
print('[CONFIGURE TENSORFLOW NOTEBOOK INSTANCE]')
params = "--hostname {0} --keyfile {1} " \
"--region {2} --os_user {3} " \
- "--ip_adress {4} --exploratory_name {5}" \
+ "--ip_adress {4} --exploratory_name {5} --edge_ip {6}" \
.format(instance_hostname, keyfile_name,
os.environ['aws_region'], notebook_config['dlab_ssh_user'],
- notebook_config['ip_address'], notebook_config['exploratory_name'])
+ notebook_config['ip_address'], notebook_config['exploratory_name'], edge_ip)
try:
local("~/scripts/{}.py {}".format('configure_tensor_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
index b59cf37..4fdde32 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/zeppelin_configure.py
@@ -88,6 +88,7 @@ if __name__ == "__main__":
else:
edge_instance_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Public')
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_ip = get_instance_ip_address(notebook_config['tag_name'], edge_instance_name).get('Private')
try:
if os.environ['conf_os_family'] == 'debian':
@@ -167,14 +168,14 @@ if __name__ == "__main__":
"--zeppelin_version {10} --scala_version {11} " \
"--livy_version {12} --multiple_clusters {13} " \
"--r_mirror {14} --endpoint_url {15} " \
- "--ip_adress {16} --exploratory_name {17}" \
+ "--ip_adress {16} --exploratory_name {17} --edge_ip {18}" \
.format(instance_hostname, notebook_config['instance_name'], keyfile_name, os.environ['aws_region'],
json.dumps(additional_config), notebook_config['dlab_ssh_user'], os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], edge_instance_hostname, '3128',
os.environ['notebook_zeppelin_version'], os.environ['notebook_scala_version'],
os.environ['notebook_livy_version'], os.environ['notebook_multiple_clusters'],
os.environ['notebook_r_mirror'], endpoint_url, notebook_config['ip_address'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_ip)
try:
local("~/scripts/{}.py {}".format('configure_zeppelin_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
index a8f7492..4c2e037 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/deeplearning_configure.py
@@ -81,6 +81,8 @@ if __name__ == "__main__":
edge_instance_hostname = AzureMeta().get_instance_public_ip_address(notebook_config['resource_group_name'],
edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_hostname = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
+ edge_instance_name)
if os.environ['conf_os_family'] == 'debian':
initial_user = 'ubuntu'
@@ -173,12 +175,12 @@ if __name__ == "__main__":
"--os_user {2} --jupyter_version {3} " \
"--scala_version {4} --spark_version {5} " \
"--hadoop_version {6} --region {7} " \
- "--r_mirror {8} --ip_adress {9} --exploratory_name {10}" \
+ "--r_mirror {8} --ip_adress {9} --exploratory_name {10} --edge_ip {11}" \
.format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'],
os.environ['notebook_jupyter_version'], os.environ['notebook_scala_version'],
os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'],
os.environ['azure_region'], os.environ['notebook_r_mirror'],
- notebook_config['ip_address'], notebook_config['exploratory_name'])
+ notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_deep_learning_node', params))
remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index 2bce792..2ba2501 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -80,6 +80,8 @@ if __name__ == "__main__":
edge_instance_hostname = AzureMeta().get_instance_public_ip_address(notebook_config['resource_group_name'],
edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_hostname = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
+ edge_instance_name)
if os.environ['conf_os_family'] == 'debian':
initial_user = 'ubuntu'
@@ -156,12 +158,12 @@ if __name__ == "__main__":
"--region {2} --spark_version {3} " \
"--hadoop_version {4} --os_user {5} " \
"--scala_version {6} --r_mirror {7} " \
- "--ip_adress {8} --exploratory_name {9}".\
+ "--ip_adress {8} --exploratory_name {9} --edge_ip {10}".\
format(instance_hostname, keyfile_name,
os.environ['azure_region'], os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], notebook_config['dlab_ssh_user'],
os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'],
- notebook_config['ip_address'], notebook_config['exploratory_name'])
+ notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_jupyter_node', params))
remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
diff --git a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
index 4f466ce..fdb620e 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/rstudio_configure.py
@@ -82,6 +82,8 @@ if __name__ == "__main__":
edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['rstudio_pass'] = id_generator()
+ edge_hostname = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
+ edge_instance_name)
if os.environ['conf_os_family'] == 'debian':
initial_user = 'ubuntu'
@@ -157,12 +159,12 @@ if __name__ == "__main__":
params = "--hostname {0} --keyfile {1} " \
"--region {2} --rstudio_pass {3} " \
"--rstudio_version {4} --os_user {5} " \
- "--r_mirror {6} --ip_adress {7} --exploratory_name {8}" \
+ "--r_mirror {6} --ip_adress {7} --exploratory_name {8} --edge_ip" \
.format(instance_hostname, keyfile_name,
os.environ['azure_region'], notebook_config['rstudio_pass'],
os.environ['notebook_rstudio_version'], notebook_config['dlab_ssh_user'],
os.environ['notebook_r_mirror'], notebook_config['ip_address'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_rstudio_node', params))
remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
diff --git a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
index c6d51eb..e40a14a 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/tensor_configure.py
@@ -83,6 +83,8 @@ if __name__ == "__main__":
edge_instance_hostname = AzureMeta().get_instance_public_ip_address(notebook_config['resource_group_name'],
edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_hostname = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
+ edge_instance_name)
if os.environ['conf_os_family'] == 'debian':
initial_user = 'ubuntu'
@@ -155,10 +157,10 @@ if __name__ == "__main__":
print('[CONFIGURE TENSORFLOW NOTEBOOK INSTANCE]')
params = "--hostname {0} --keyfile {1} " \
"--region {2} --os_user {3} " \
- "--ip_adress {4} --exploratory_name {5}" \
+ "--ip_adress {4} --exploratory_name {5} --edge_ip {6}" \
.format(instance_hostname, keyfile_name,
os.environ['azure_region'], notebook_config['dlab_ssh_user'],
- notebook_config['ip_address'], notebook_config['exploratory_name'])
+ notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_tensor_node', params))
remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
diff --git a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
index 5917ee2..f570dcb 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/zeppelin_configure.py
@@ -82,6 +82,8 @@ if __name__ == "__main__":
edge_instance_hostname = AzureMeta().get_instance_public_ip_address(notebook_config['resource_group_name'],
edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
+ edge_hostname = AzureMeta().get_private_ip_address(notebook_config['resource_group_name'],
+ edge_instance_name)
if os.environ['conf_os_family'] == 'debian':
initial_user = 'ubuntu'
@@ -166,14 +168,14 @@ if __name__ == "__main__":
"--zeppelin_version {10} --scala_version {11} " \
"--livy_version {12} --multiple_clusters {13} " \
"--r_mirror {14} --endpoint_url {15} " \
- "--ip_adress {16} --exploratory_name {17}" \
+ "--ip_adress {16} --exploratory_name {17} --edge_ip" \
.format(instance_hostname, notebook_config['instance_name'], keyfile_name, os.environ['azure_region'],
json.dumps(additional_config), notebook_config['dlab_ssh_user'], os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], edge_instance_private_hostname, '3128',
os.environ['notebook_zeppelin_version'], os.environ['notebook_scala_version'],
os.environ['notebook_livy_version'], os.environ['notebook_multiple_clusters'],
os.environ['notebook_r_mirror'], 'null',
- notebook_config['ip_address'], notebook_config['exploratory_name'])
+ notebook_config['ip_address'], notebook_config['exploratory_name'], edge_hostname)
try:
local("~/scripts/{}.py {}".format('configure_zeppelin_node', params))
remount_azure_disk(True, notebook_config['dlab_ssh_user'], instance_hostname,
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
index 088316d..6d4637b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
@@ -58,6 +58,7 @@ if __name__ == "__main__":
instance_hostname = GCPMeta().get_private_ip_address(notebook_config['instance_name'])
edge_instance_name = '{0}-{1}-edge'.format(notebook_config['service_base_name'], notebook_config['edge_user_name'])
edge_instance_hostname = GCPMeta().get_instance_public_ip_by_name(edge_instance_name)
+ edge_instance_private_ip = GCPMeta().get_private_ip_address(edge_instance_name)
notebook_config['ssh_key_path'] = '{0}{1}.pem'.format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
notebook_config['dlab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
@@ -130,12 +131,12 @@ if __name__ == "__main__":
"--os_user {} --jupyter_version {} " \
"--scala_version {} --spark_version {} " \
"--hadoop_version {} --region {} " \
- "--r_mirror {} --exploratory_name {}" \
+ "--r_mirror {} --exploratory_name {} --edge_ip" \
.format(instance_hostname, notebook_config['ssh_key_path'], notebook_config['dlab_ssh_user'],
os.environ['notebook_jupyter_version'], os.environ['notebook_scala_version'],
os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'],
os.environ['gcp_region'], os.environ['notebook_r_mirror'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('configure_deep_learning_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index ad4f8f1..0a452a9 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -132,12 +132,13 @@ if __name__ == "__main__":
"--region {} --spark_version {} " \
"--hadoop_version {} --os_user {} " \
"--scala_version {} --r_mirror {} " \
- "--exploratory_name {}".\
+ "--exploratory_name {} "\
+ "--edge_ip {}".\
format(instance_hostname, notebook_config['ssh_key_path'],
os.environ['gcp_region'], os.environ['notebook_spark_version'],
os.environ['notebook_hadoop_version'], notebook_config['dlab_ssh_user'],
os.environ['notebook_scala_version'], os.environ['notebook_r_mirror'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('configure_jupyter_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
index 8d9da73..0d6708e 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
@@ -136,12 +136,12 @@ if __name__ == "__main__":
params = "--hostname {0} --keyfile {1} " \
"--region {2} --rstudio_pass {3} " \
"--rstudio_version {4} --os_user {5} " \
- "--r_mirror {6} --ip_adress {7} --exploratory_name {8}" \
+ "--r_mirror {6} --ip_adress {7} --exploratory_name {8} --edge_ip {9}" \
.format(instance_hostname, notebook_config['ssh_key_path'],
os.environ['gcp_region'], notebook_config['rstudio_pass'],
os.environ['notebook_rstudio_version'], notebook_config['dlab_ssh_user'],
os.environ['notebook_r_mirror'], notebook_config['ip_address'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('configure_rstudio_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
index 0955584..e00a47f 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
@@ -140,11 +140,11 @@ if __name__ == "__main__":
params = "--hostname {} --keyfile {} " \
"--region {} --rstudio_pass {} " \
"--rstudio_version {} --os_user {} " \
- "--r_mirror {} --exploratory_name {}" \
+ "--r_mirror {} --exploratory_name {} --edge_ip {}" \
.format(instance_hostname, notebook_config['ssh_key_path'],
os.environ['gcp_region'], notebook_config['rstudio_pass'],
os.environ['notebook_rstudio_version'], notebook_config['dlab_ssh_user'],
- os.environ['notebook_r_mirror'], notebook_config['exploratory_name'])
+ os.environ['notebook_r_mirror'], notebook_config['exploratory_name'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('configure_tensor-rstudio_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
index c646f14..8d69ee2 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
@@ -131,10 +131,10 @@ if __name__ == "__main__":
try:
logging.info('[CONFIGURE TENSORFLOW NOTEBOOK INSTANCE]')
print('[CONFIGURE TENSORFLOW NOTEBOOK INSTANCE]')
- params = "--hostname {} --keyfile {} --region {} --os_user {} --exploratory_name {}" \
+ params = "--hostname {} --keyfile {} --region {} --os_user {} --exploratory_name {} --edge_ip {}" \
.format(instance_hostname, notebook_config['ssh_key_path'],
os.environ['gcp_region'], notebook_config['dlab_ssh_user'],
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('configure_tensor_node', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
index 1b3602b..e742663 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
@@ -143,14 +143,15 @@ if __name__ == "__main__":
"--zeppelin_version {} --scala_version {} " \
"--livy_version {} --multiple_clusters {} " \
"--r_mirror {} --endpoint_url {} " \
- "--exploratory_name {}" \
+ "--exploratory_name {} " \
+ "--edge_ip {}" \
.format(instance_hostname, notebook_config['instance_name'], notebook_config['ssh_key_path'],
os.environ['gcp_region'], json.dumps(additional_config), notebook_config['dlab_ssh_user'],
os.environ['notebook_spark_version'], os.environ['notebook_hadoop_version'], edge_instance_name,
'3128', os.environ['notebook_zeppelin_version'], os.environ['notebook_scala_version'],
os.environ['notebook_livy_version'], os.environ['notebook_multiple_clusters'],
os.environ['notebook_r_mirror'], 'null',
- notebook_config['exploratory_name'])
+ notebook_config['exploratory_name'], edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('configure_zeppelin_node', params))
except:
diff --git a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
index 96bebb9..064d492 100644
--- a/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
+++ b/infrastructure-provisioning/src/jupyter/scripts/configure_jupyter_node.py
@@ -40,6 +40,7 @@ parser.add_argument('--scala_version', type=str, default='')
parser.add_argument('--r_mirror', type=str, default='')
parser.add_argument('--ip_adress', type=str, default='')
parser.add_argument('--exploratory_name', type=str, default='')
+parser.add_argument('--edge_ip', type=str, default='')
args = parser.parse_args()
spark_version = args.spark_version
@@ -126,7 +127,7 @@ if __name__ == "__main__":
print("Install nodejs")
install_nodejs(args.os_user)
print("Install ungit")
- install_ungit(args.os_user, args.exploratory_name)
+ install_ungit(args.os_user, args.exploratory_name, args.edge_ip)
if exists('/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
install_gitlab_cert(args.os_user, gitlab_certfile)
diff --git a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
index de1b7a4..a2878b8 100644
--- a/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
+++ b/infrastructure-provisioning/src/rstudio/scripts/configure_rstudio_node.py
@@ -42,6 +42,7 @@ parser.add_argument('--rstudio_version', type=str, default='')
parser.add_argument('--r_mirror', type=str, default='')
parser.add_argument('--ip_adress', type=str, default='')
parser.add_argument('--exploratory_name', type=str, default='')
+parser.add_argument('--edge_ip', type=str, default='')
args = parser.parse_args()
spark_version = os.environ['notebook_spark_version']
@@ -106,7 +107,7 @@ if __name__ == "__main__":
print("Install nodejs")
install_nodejs(args.os_user)
print("Install Ungit")
- install_ungit(args.os_user, args.exploratory_name)
+ install_ungit(args.os_user, args.exploratory_name, args.edge_ip)
if exists('/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
install_gitlab_cert(args.os_user, gitlab_certfile)
diff --git a/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py b/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
index 09bc542..fb579ad 100644
--- a/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
+++ b/infrastructure-provisioning/src/tensor-rstudio/scripts/configure_tensor-rstudio_node.py
@@ -42,6 +42,7 @@ parser.add_argument('--rstudio_version', type=str, default='')
parser.add_argument('--r_mirror', type=str, default='')
parser.add_argument('--ip_adress', type=str, default='')
parser.add_argument('--exploratory_name', type=str, default='')
+parser.add_argument('--edge_ip', type=str, default='')
args = parser.parse_args()
spark_version = os.environ['notebook_spark_version']
@@ -124,7 +125,7 @@ if __name__ == "__main__":
print("Install nodejs")
install_nodejs(args.os_user)
print("Install Ungit")
- install_ungit(args.os_user, args.exploratory_name)
+ install_ungit(args.os_user, args.exploratory_name, args.edge_ip)
if exists('/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
install_gitlab_cert(args.os_user, gitlab_certfile)
diff --git a/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py b/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
index 85934c2..43e220e 100644
--- a/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
+++ b/infrastructure-provisioning/src/tensor/scripts/configure_tensor_node.py
@@ -39,6 +39,7 @@ parser.add_argument('--region', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
parser.add_argument('--ip_adress', type=str, default='')
parser.add_argument('--exploratory_name', type=str, default='')
+parser.add_argument('--edge_ip', type=str, default='')
args = parser.parse_args()
spark_version = os.environ['notebook_spark_version']
@@ -127,7 +128,7 @@ if __name__ == "__main__":
print("Install nodejs")
install_nodejs(args.os_user)
print("Install Ungit")
- install_ungit(args.os_user, args.exploratory_name)
+ install_ungit(args.os_user, args.exploratory_name, args.edge_ip)
if exists('/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
install_gitlab_cert(args.os_user, gitlab_certfile)
diff --git a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
index c803622..457b075 100644
--- a/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
+++ b/infrastructure-provisioning/src/zeppelin/scripts/configure_zeppelin_node.py
@@ -52,6 +52,7 @@ parser.add_argument('--r_mirror', type=str, default='')
parser.add_argument('--endpoint_url', type=str, default='')
parser.add_argument('--ip_adress', type=str, default='')
parser.add_argument('--exploratory_name', type=str, default='')
+parser.add_argument('--edge_ip', type=str, default='')
args = parser.parse_args()
spark_version = args.spark_version
@@ -243,7 +244,7 @@ if __name__ == "__main__":
print("Install nodejs")
install_nodejs(args.os_user)
print("Install Ungit")
- install_ungit(args.os_user, args.exploratory_name)
+ install_ungit(args.os_user, args.exploratory_name, args.edge_ip)
if exists('/home/{0}/{1}'.format(args.os_user, gitlab_certfile)):
install_gitlab_cert(args.os_user, gitlab_certfile)
# COPY PRE-COMMIT SCRIPT TO ZEPPELIN
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org