You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/06/07 19:25:20 UTC
[incubator-dlab] 02/03: [DLAB-1835]- Private IP instead hostname
for ssh connection changed
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1835
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit bd89d6f70f3afb26debfcc684d8ffef9c0fe6a0a
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Sun Jun 7 22:24:33 2020 +0300
[DLAB-1835]- Private IP instead hostname for ssh connection changed
---
.../src/general/conf/dlab.ini | 2 +
.../general/scripts/aws/common_prepare_notebook.py | 5 +-
.../src/general/scripts/aws/edge_configure.py | 18 ++++---
.../src/general/scripts/aws/jupyter_configure.py | 18 +++----
.../src/general/scripts/aws/project_prepare.py | 55 +++++++++++++---------
5 files changed, 57 insertions(+), 41 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 8ab5f9e..0aedd0b 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -94,6 +94,8 @@ private_subnet_prefix = 24
# subnet_id =
### Id of the subnet for notebooks and compute engines
# subnet2_id =
+### CIDR of the subnet for notebooks and compute engines
+# private_subnet_cidr =
### Id of the vpc for whole dlab provisioning
# vpc_id =
### Id of the secondary vpc for notebooks and compute engines
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
index 5c481ac..9a9bf51 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_prepare_notebook.py
@@ -112,7 +112,10 @@ if __name__ == "__main__":
tag = {"Key": notebook_config['tag_name'],
"Value": "{}-{}-{}-subnet".format(notebook_config['service_base_name'], notebook_config['project_name'],
notebook_config['endpoint_name'])}
- notebook_config['subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag)
+ if os.environ['aws_private_subnet_cidr']:
+ notebook_config['subnet_cidr'] = os.environ['aws_private_subnet_cidr']
+ else:
+ notebook_config['subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
with open('/root/result.json', 'w') as f:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
index cfb4aa9..3a30bbd 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/edge_configure.py
@@ -110,11 +110,9 @@ if __name__ == "__main__":
for cidr in os.environ['conf_allowed_ip_cidr'].split(','):
edge_conf['allowed_ip_cidr'].append(cidr.replace(' ', ''))
- if edge_conf['network_type'] == 'public':
- edge_conf['instance_hostname'] = dlab.meta_lib.get_instance_hostname(edge_conf['tag_name'],
+ edge_conf['instance_hostname'] = dlab.meta_lib.get_instance_hostname(edge_conf['tag_name'],
edge_conf['instance_name'])
- elif edge_conf['network_type'] == 'private':
- edge_conf['instance_hostname'] = dlab.meta_lib.get_instance_ip_address(
+ edge_conf['instance_private_ip'] = dlab.meta_lib.get_instance_ip_address(
edge_conf['tag_name'], edge_conf['instance_name']).get('Private')
edge_conf['keyfile_name'] = "{}{}.pem".format(os.environ['conf_key_dir'], edge_conf['key_name'])
@@ -142,7 +140,7 @@ if __name__ == "__main__":
logging.info('[CREATING DLAB SSH USER]')
print('[CREATING DLAB SSH USER]')
params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format(
- edge_conf['instance_hostname'], os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem",
+ edge_conf['instance_private_ip'], os.environ['conf_key_dir'] + os.environ['conf_key_name'] + ".pem",
edge_conf['initial_user'], edge_conf['dlab_ssh_user'], edge_conf['sudo_group'])
try:
@@ -159,7 +157,7 @@ if __name__ == "__main__":
print('[INSTALLING PREREQUISITES]')
logging.info('[INSTALLING PREREQUISITES]')
params = "--hostname {} --keyfile {} --user {} --region {}".\
- format(edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'],
+ format(edge_conf['instance_private_ip'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'],
os.environ['aws_region'])
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@ -184,7 +182,7 @@ if __name__ == "__main__":
"vpc_cidrs": edge_conf['vpc_cidrs'],
"allowed_ip_cidr": edge_conf['allowed_ip_cidr']}
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
- edge_conf['instance_hostname'], edge_conf['keyfile_name'], json.dumps(additional_config),
+ edge_conf['instance_private_ip'], edge_conf['keyfile_name'], json.dumps(additional_config),
edge_conf['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('configure_http_proxy', params))
@@ -204,7 +202,7 @@ if __name__ == "__main__":
"user_keydir": os.environ['conf_key_dir'],
"user_key": edge_conf['user_key']}
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
- edge_conf['instance_hostname'], edge_conf['keyfile_name'], json.dumps(additional_config),
+ edge_conf['instance_private_ip'], edge_conf['keyfile_name'], json.dumps(additional_config),
edge_conf['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('install_user_key', params))
@@ -222,7 +220,7 @@ if __name__ == "__main__":
edge_conf['keycloak_client_secret'] = str(uuid.uuid4())
params = "--hostname {} --keyfile {} --user {} --keycloak_client_id {} --keycloak_client_secret {} " \
"--step_cert_sans '{}' ".format(
- edge_conf['instance_hostname'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'],
+ edge_conf['instance_private_ip'], edge_conf['keyfile_name'], edge_conf['dlab_ssh_user'],
'{}-{}-{}'.format(edge_conf['service_base_name'], edge_conf['project_name'],
edge_conf['endpoint_name']),
edge_conf['keycloak_client_secret'], edge_conf['step_cert_sans'])
@@ -237,7 +235,7 @@ if __name__ == "__main__":
edge_conf['service_base_name'], os.environ['keycloak_auth_server_url'],
os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
os.environ['keycloak_user_password'], edge_conf['keycloak_client_secret'],
- edge_conf['instance_hostname'], edge_conf['instance_hostname'], edge_conf['project_name'],
+ edge_conf['instance_hostname'], edge_conf['instance_private_ip'], edge_conf['project_name'],
edge_conf['endpoint_name'])
try:
local("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index cc53b22..bb48bcf 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@ -93,6 +93,8 @@ if __name__ == "__main__":
edge_instance_hostname = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'], edge_instance_name)
edge_instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
edge_instance_name).get('Private')
+ instance_private_ip = dlab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
+ notebook_config['instance_name']).get('Private')
notebook_config['edge_instance_hostname'] = dlab.meta_lib.get_instance_hostname(notebook_config['tag_name'],
edge_instance_name)
keyfile_name = "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name'])
@@ -113,7 +115,7 @@ if __name__ == "__main__":
logging.info('[CREATING DLAB SSH USER]')
print('[CREATING DLAB SSH USER]')
params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format(
- instance_hostname, "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
+ instance_private_ip, "{}{}.pem".format(os.environ['conf_key_dir'], os.environ['conf_key_name']),
notebook_config['initial_user'], notebook_config['dlab_ssh_user'], notebook_config['sudo_group'])
try:
@@ -132,7 +134,7 @@ if __name__ == "__main__":
print('[CONFIGURE PROXY ON JUPYTER INSTANCE]')
additional_config = {"proxy_host": edge_instance_hostname, "proxy_port": "3128"}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}".format(
- instance_hostname, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config),
+ instance_private_ip, notebook_config['instance_name'], keyfile_name, json.dumps(additional_config),
notebook_config['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('common_configure_proxy', params))
@@ -149,7 +151,7 @@ if __name__ == "__main__":
logging.info('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
print('[INSTALLING PREREQUISITES TO JUPYTER NOTEBOOK INSTANCE]')
params = "--hostname {} --keyfile {} --user {} --region {} --edge_private_ip {}".format(
- instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
+ instance_private_ip, keyfile_name, notebook_config['dlab_ssh_user'], os.environ['aws_region'],
edge_instance_private_ip)
try:
local("~/scripts/{}.py {}".format('install_prerequisites', params))
@@ -176,7 +178,7 @@ if __name__ == "__main__":
"--ip_address {8} " \
"--exploratory_name {9} " \
"--edge_ip {10}".\
- format(instance_hostname,
+ format(instance_private_ip,
keyfile_name,
os.environ['aws_region'],
os.environ['notebook_spark_version'],
@@ -203,7 +205,7 @@ if __name__ == "__main__":
additional_config = {"user_keyname": notebook_config['user_keyname'],
"user_keydir": os.environ['conf_key_dir']}
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
- instance_hostname, keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user'])
+ instance_private_ip, keyfile_name, json.dumps(additional_config), notebook_config['dlab_ssh_user'])
try:
local("~/scripts/{}.py {}".format('install_user_key', params))
except:
@@ -218,7 +220,7 @@ if __name__ == "__main__":
print('[SETUP USER GIT CREDENTIALS]')
logging.info('[SETUP USER GIT CREDENTIALS]')
params = '--os_user {} --notebook_ip {} --keyfile "{}"' \
- .format(notebook_config['dlab_ssh_user'], instance_hostname, keyfile_name)
+ .format(notebook_config['dlab_ssh_user'], instance_private_ip, keyfile_name)
try:
local("~/scripts/{}.py {}".format('common_download_git_certfile', params))
local("~/scripts/{}.py {}".format('manage_git_creds', params))
@@ -235,7 +237,7 @@ if __name__ == "__main__":
print('[POST CONFIGURING PROCESS')
if notebook_config['notebook_image_name'] not in [notebook_config['expected_image_name'], 'None']:
params = "--hostname {} --keyfile {} --os_user {} --nb_tag_name {} --nb_tag_value {}" \
- .format(instance_hostname, keyfile_name, notebook_config['dlab_ssh_user'],
+ .format(instance_private_ip, keyfile_name, notebook_config['dlab_ssh_user'],
notebook_config['tag_name'], notebook_config['instance_name'])
try:
local("~/scripts/{}.py {}".format('common_remove_remote_kernels', params))
@@ -260,7 +262,7 @@ if __name__ == "__main__":
"--type {} " \
"--exploratory_name {} " \
"--additional_info '{}'"\
- .format(edge_instance_hostname,
+ .format(edge_instance_private_ip,
keyfile_name,
notebook_config['dlab_ssh_user'],
'jupyter',
diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index 9d44ba5..0523fbc 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -184,34 +184,45 @@ if __name__ == "__main__":
except KeyError:
project_conf['vpc2_id'] = project_conf['vpc_id']
+ tag = {"Key": project_conf['tag_name'],
+ "Value": "{0}-{1}-{2}-subnet".format(project_conf['service_base_name'], project_conf['project_name'],
+ project_conf['endpoint_name'])}
+
+ if os.environ['aws_private_subnet_cidr']:
+ project_conf['private_subnet_cidr'] = os.environ['aws_private_subnet_cidr']
+ else:
+ try:
+ try:
+ logging.info('[CREATE SUBNET]')
+ print('[CREATE SUBNET]')
+ params = "--vpc_id '{}' --infra_tag_name {} --infra_tag_value {} --prefix {} " \
+ "--user_subnets_range '{}' --subnet_name {} --zone {}".format(
+ project_conf['vpc2_id'], project_conf['tag_name'], project_conf['service_base_name'],
+ project_conf['private_subnet_prefix'], os.environ['conf_user_subnets_range'],
+ project_conf['private_subnet_name'],
+ project_conf['zone'])
+ try:
+ local("~/scripts/{}.py {}".format('common_create_subnet', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ dlab.fab.append_result("Failed to create subnet.", str(err))
+ sys.exit(1)
+ project_conf['private_subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag)
+ except Exception as err:
+ dlab.fab.append_result("Failed to create subnet.", str(err))
+ sys.exit(1)
try:
- logging.info('[CREATE SUBNET]')
- print('[CREATE SUBNET]')
- params = "--vpc_id '{}' --infra_tag_name {} --infra_tag_value {} --prefix {} " \
- "--user_subnets_range '{}' --subnet_name {} --zone {}".format(
- project_conf['vpc2_id'], project_conf['tag_name'], project_conf['service_base_name'],
- project_conf['private_subnet_prefix'], os.environ['conf_user_subnets_range'],
- project_conf['private_subnet_name'],
- project_conf['zone'])
- try:
- local("~/scripts/{}.py {}".format('common_create_subnet', params))
- except:
- traceback.print_exc()
- raise Exception
+ subnet_id = dlab.meta_lib.get_subnet_by_cidr(project_conf['private_subnet_cidr'], project_conf['vpc2_id'])
+ print('Subnet id: {}'.format(subnet_id))
+ print('NEW SUBNET CIDR CREATED: {}'.format(project_conf['private_subnet_cidr']))
except Exception as err:
- dlab.fab.append_result("Failed to create subnet.", str(err))
+ dlab.fab.append_result("Failed to assign subnet.", str(err))
sys.exit(1)
- tag = {"Key": project_conf['tag_name'],
- "Value": "{0}-{1}-{2}-subnet".format(project_conf['service_base_name'], project_conf['project_name'],
- project_conf['endpoint_name'])}
- project_conf['private_subnet_cidr'] = dlab.meta_lib.get_subnet_by_tag(tag)
- subnet_id = dlab.meta_lib.get_subnet_by_cidr(project_conf['private_subnet_cidr'], project_conf['vpc2_id'])
- print('Subnet id: {}'.format(subnet_id))
- print('NEW SUBNET CIDR CREATED: {}'.format(project_conf['private_subnet_cidr']))
-
try:
logging.info('[CREATE EDGE ROLES]')
print('[CREATE EDGE ROLES]')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org