You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/02/04 10:36:18 UTC
[incubator-datalab] branch DATALAB-2674 created (now 986f608)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 986f608 [DATALAB-2674]: added optional parameter to enable bucket versioning
This branch includes the following new commits:
new c8288ad [DATALAB-2674]: added os login for gcp
new 7e28228 [DATALAB-2674]: added block project ssh keys for gcp instances
new aee33ee [DATALAB-2674]: moved create_instance arguments to new line
new 986f608 [DATALAB-2674]: added optional parameter to enable bucket versioning
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/04: [DATALAB-2674]: moved create_instance arguments to new line
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit aee33eeab0af754610584adfa1a8ac704234b9fd
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Feb 1 14:19:34 2022 +0200
[DATALAB-2674]: moved create_instance arguments to new line
---
infrastructure-provisioning/src/general/lib/gcp/actions_lib.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 025f1b5..69eb9e7 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -320,7 +320,8 @@ class GCPActions:
initial_user, image_name, secondary_image_name, service_account_name, instance_class,
network_tag, labels, static_ip='',
primary_disk_size='12', secondary_disk_size='30',
- gpu_accelerator_type='None', gpu_accelerator_count='1', os_login_enabled='FALSE', block_project_ssh_keys='FALSE'):
+ gpu_accelerator_type='None', gpu_accelerator_count='1',
+ os_login_enabled='FALSE', block_project_ssh_keys='FALSE'):
key = RSA.importKey(open(ssh_key_path, 'rb').read())
ssh_key = key.publickey().exportKey("OpenSSH").decode('UTF-8')
unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/04: [DATALAB-2674]: added os login for gcp
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c8288ad57a8a7ffe7a5eec330833eac5d94eeb9c
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Feb 1 11:42:54 2022 +0200
[DATALAB-2674]: added os login for gcp
---
infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++
infrastructure-provisioning/src/general/conf/datalab.ini | 2 ++
.../src/general/lib/gcp/actions_lib.py | 6 +++++-
.../src/general/scripts/gcp/common_create_instance.py | 3 ++-
.../src/general/scripts/gcp/common_prepare_notebook.py | 7 +++++--
.../src/general/scripts/gcp/dataengine_prepare.py | 12 ++++++++----
.../src/general/scripts/gcp/project_prepare.py | 7 +++++--
.../src/general/scripts/gcp/ssn_prepare.py | 7 +++++--
8 files changed, 34 insertions(+), 12 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index 9dd70e4..c039834 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -259,6 +259,8 @@ def build_parser():
help='One of more comma-separated GCP Firewall rules for SSN')
gcp_parser.add_argument('--gcp_ssn_instance_size', type=str, default='n1-standard-2',
help='The SSN instance shape')
+ gcp_parser.add_argument('--gcp_os_login_enabled', type=str, default='FALSE',
+ help='TRUE if os login enabled for gcp instances')
gcp_required_args = gcp_parser.add_argument_group('Required arguments')
gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region')
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 26145fe..73ed7ba 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -228,6 +228,8 @@ datalake_enable = false
# subnet_name =
### Names of the firewall rules for SSN
# firewall_rules =
+### True if os login is enabled for instances
+# os_login_enabled =
### GCP region name for whole DataLab provisioning
region = us-west1
### GCP zone name for whole DataLab provisioning
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index e4b070f..37e8d30 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -320,7 +320,7 @@ class GCPActions:
initial_user, image_name, secondary_image_name, service_account_name, instance_class,
network_tag, labels, static_ip='',
primary_disk_size='12', secondary_disk_size='30',
- gpu_accelerator_type='None', gpu_accelerator_count='1'):
+ gpu_accelerator_type='None', gpu_accelerator_count='1', os_login_enabled='FALSE'):
key = RSA.importKey(open(ssh_key_path, 'rb').read())
ssh_key = key.publickey().exportKey("OpenSSH").decode('UTF-8')
unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
@@ -424,6 +424,10 @@ class GCPActions:
{
"key": "ssh-keys",
"value": "{}:{}".format(initial_user, ssh_key)
+ },
+ {
+ "key": "enable-oslogin",
+ "value": "{}".format(os_login_enabled)
}
]
},
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
index d780b44..246675d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
@@ -50,6 +50,7 @@ parser.add_argument('--gpu_accelerator_count', type=str, default='None')
parser.add_argument('--network_tag', type=str, default='')
parser.add_argument('--cluster_name', type=str, default='')
parser.add_argument('--service_base_name', type=str, default='')
+parser.add_argument('--os_login_enabled', type=str, default='FALSE')
args = parser.parse_args()
@@ -65,7 +66,7 @@ if __name__ == "__main__":
args.secondary_image_name, args.service_account_name, args.instance_class,
args.network_tag, json.loads(args.labels), args.static_ip,
args.primary_disk_size, args.secondary_disk_size, args.gpu_accelerator_type,
- args.gpu_accelerator_count)
+ args.gpu_accelerator_count, args.os_login_enabled)
else:
parser.print_help()
sys.exit(2)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
index dbacfab..9954592 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
@@ -148,6 +148,9 @@ if __name__ == "__main__":
notebook_config['secondary_image_name'] = 'global/images/{}'.format(
notebook_config['secondary_image_name'].get('name'))
+ if os.environ['gcp_os_login_enabled'] != 'FALSE':
+ notebook_config['gcp_os_login_enabled'] = 'TRUE'
+
notebook_config['gpu_accelerator_type'] = 'None'
notebook_config['gpu_accelerator_count'] = 'None'
@@ -193,7 +196,7 @@ if __name__ == "__main__":
"--ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9} " \
"--secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \
"--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --labels '{17}' " \
- "--service_base_name {18}".\
+ "--service_base_name {18} --os_login_enabled {19}".\
format(notebook_config['instance_name'], notebook_config['region'], notebook_config['zone'],
notebook_config['vpc_name'], notebook_config['subnet_name'], notebook_config['instance_size'],
notebook_config['ssh_key_path'], notebook_config['initial_user'],
@@ -201,7 +204,7 @@ if __name__ == "__main__":
notebook_config['secondary_image_name'], 'notebook', notebook_config['primary_disk_size'],
notebook_config['secondary_disk_size'], notebook_config['gpu_accelerator_type'],
notebook_config['gpu_accelerator_count'], notebook_config['network_tag'],
- json.dumps(notebook_config['labels']), notebook_config['service_base_name'])
+ json.dumps(notebook_config['labels']), notebook_config['service_base_name'], notebook_config['gcp_os_login_enabled'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
index 84ee186..643354d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
@@ -97,6 +97,10 @@ if __name__ == "__main__":
if os.environ['conf_os_family'] == 'redhat':
initial_user = 'ec2-user'
sudo_group = 'wheel'
+
+ if os.environ['gcp_os_login_enabled'] != 'FALSE':
+ data_engine['gcp_os_login_enabled'] = 'TRUE'
+
data_engine['cluster_name'] = "{}-{}-{}-de-{}".format(data_engine['service_base_name'],
data_engine['project_name'],
data_engine['endpoint_name'],
@@ -189,14 +193,14 @@ if __name__ == "__main__":
"--ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9} " \
"--secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \
"--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --cluster_name {17} " \
- "--labels '{18}' --service_base_name {19}". \
+ "--labels '{18}' --service_base_name {19} --os_login_enabled {20}". \
format(data_engine['master_node_name'], data_engine['region'], data_engine['zone'], data_engine['vpc_name'],
data_engine['subnet_name'], data_engine['master_size'], data_engine['ssh_key_path'], initial_user,
data_engine['dataengine_service_account_name'], data_engine['primary_image_name'],
data_engine['secondary_image_name'], 'dataengine', data_engine['primary_disk_size'],
data_engine['secondary_disk_size'], data_engine['gpu_master_accelerator_type'],
data_engine['gpu_master_accelerator_count'], data_engine['network_tag'], data_engine['cluster_name'],
- json.dumps(data_engine['master_labels']), data_engine['service_base_name'])
+ json.dumps(data_engine['master_labels']), data_engine['service_base_name'], data_engine['gcp_os_login_enabled'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
@@ -215,7 +219,7 @@ if __name__ == "__main__":
"--instance_size {5} --ssh_key_path {6} --initial_user {7} --service_account_name {8} " \
"--image_name {9} --secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \
"--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --cluster_name {17} " \
- "--labels '{18}' --service_base_name {19}". \
+ "--labels '{18}' --service_base_name {19} --os_login_enabled {20}". \
format(slave_name, data_engine['region'], data_engine['zone'],
data_engine['vpc_name'], data_engine['subnet_name'], data_engine['slave_size'],
data_engine['ssh_key_path'], initial_user, data_engine['dataengine_service_account_name'],
@@ -224,7 +228,7 @@ if __name__ == "__main__":
data_engine['secondary_disk_size'], data_engine['gpu_slave_accelerator_type'],
data_engine['gpu_slave_accelerator_count'], data_engine['network_tag'],
data_engine['cluster_name'], json.dumps(data_engine['slave_labels']),
- data_engine['service_base_name'])
+ data_engine['service_base_name'], data_engine['gcp_os_login_enabled'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index d5a06c9..02e9667 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -500,18 +500,21 @@ if __name__ == "__main__":
project_conf['initial_user'] = 'ec2-user'
project_conf['sudo_group'] = 'wheel'
+ if os.environ['gcp_os_login_enabled'] != 'FALSE':
+ project_conf['gcp_os_login_enabled'] = 'TRUE'
+
try:
project_conf['static_ip'] = \
GCPMeta.get_static_address(project_conf['region'], project_conf['static_address_name'])['address']
logging.info('[CREATE EDGE INSTANCE]')
params = "--instance_name {} --region {} --zone {} --vpc_name {} --subnet_name {} --instance_size {} " \
"--ssh_key_path {} --initial_user {} --service_account_name {} --image_name {} --instance_class {} " \
- "--static_ip {} --network_tag {} --labels '{}' --service_base_name {}".format(
+ "--static_ip {} --network_tag {} --labels '{}' --service_base_name {} --os_login_enabled {}".format(
project_conf['instance_name'], project_conf['region'], project_conf['zone'], project_conf['vpc_name'],
project_conf['subnet_name'], project_conf['instance_size'], project_conf['ssh_key_path'],
project_conf['initial_user'], project_conf['edge_service_account_name'], project_conf['image_name'],
'edge', project_conf['static_ip'], project_conf['network_tag'],
- json.dumps(project_conf['instance_labels']), project_conf['service_base_name'])
+ json.dumps(project_conf['instance_labels']), project_conf['service_base_name'], project_conf['gcp_os_login_enabled'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
index 59f0658..9346d15 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
@@ -251,6 +251,9 @@ if __name__ == "__main__":
GCPActions.remove_vpc(ssn_conf['vpc_name'])
sys.exit(1)
+ if os.environ['gcp_os_login_enabled'] != 'FALSE':
+ ssn_conf['gcp_os_login_enabled'] = 'TRUE'
+
if os.environ['conf_os_family'] == 'debian':
ssn_conf['initial_user'] = 'ubuntu'
ssn_conf['sudo_group'] = 'sudo'
@@ -265,12 +268,12 @@ if __name__ == "__main__":
params = "--instance_name {0} --region {1} --zone {2} --vpc_name {3} --subnet_name {4} --instance_size {5}"\
" --ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9}"\
" --instance_class {10} --static_ip {11} --network_tag {12} --labels '{13}' " \
- "--primary_disk_size {14} --service_base_name {15}".\
+ "--primary_disk_size {14} --service_base_name {15} --os_login_enabled {16}".\
format(ssn_conf['instance_name'], ssn_conf['region'], ssn_conf['zone'], ssn_conf['vpc_name'],
ssn_conf['subnet_name'], ssn_conf['instance_size'], ssn_conf['ssh_key_path'],
ssn_conf['initial_user'], ssn_conf['service_account_name'], ssn_conf['image_name'], 'ssn',
ssn_conf['static_ip'], ssn_conf['network_tag'], json.dumps(ssn_conf['instance_labels']), '20',
- ssn_conf['service_base_name'])
+ ssn_conf['service_base_name'], ssn_conf['gcp_os_login_enabled'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/04: [DATALAB-2674]: added block project ssh keys for gcp instances
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 7e282286d8f98e70b2fff5e3ab7532bbdd1df163
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Feb 1 14:19:00 2022 +0200
[DATALAB-2674]: added block project ssh keys for gcp instances
---
infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++
.../src/general/lib/gcp/actions_lib.py | 6 +++++-
.../src/general/scripts/gcp/common_create_instance.py | 3 ++-
.../src/general/scripts/gcp/common_prepare_notebook.py | 8 ++++++--
.../src/general/scripts/gcp/dataengine_prepare.py | 12 ++++++++----
.../src/general/scripts/gcp/project_prepare.py | 9 +++++++--
.../src/general/scripts/gcp/ssn_prepare.py | 9 +++++++--
7 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index c039834..40b233d 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -261,6 +261,8 @@ def build_parser():
help='The SSN instance shape')
gcp_parser.add_argument('--gcp_os_login_enabled', type=str, default='FALSE',
help='TRUE if os login enabled for gcp instances')
+ gcp_parser.add_argument('--gcp_block_project_ssh_keys', type=str, default='FALSE',
+ help='TRUE to block project ssh keys for gcp instances')
gcp_required_args = gcp_parser.add_argument_group('Required arguments')
gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region')
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 37e8d30..025f1b5 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -320,7 +320,7 @@ class GCPActions:
initial_user, image_name, secondary_image_name, service_account_name, instance_class,
network_tag, labels, static_ip='',
primary_disk_size='12', secondary_disk_size='30',
- gpu_accelerator_type='None', gpu_accelerator_count='1', os_login_enabled='FALSE'):
+ gpu_accelerator_type='None', gpu_accelerator_count='1', os_login_enabled='FALSE', block_project_ssh_keys='FALSE'):
key = RSA.importKey(open(ssh_key_path, 'rb').read())
ssh_key = key.publickey().exportKey("OpenSSH").decode('UTF-8')
unique_index = datalab.meta_lib.GCPMeta().get_index_by_service_account_name(service_account_name)
@@ -428,6 +428,10 @@ class GCPActions:
{
"key": "enable-oslogin",
"value": "{}".format(os_login_enabled)
+ },
+ {
+ "key": "block-project-ssh-keys",
+ "value": "{}".format(block_project_ssh_keys)
}
]
},
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
index 246675d..adf2bf5 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_instance.py
@@ -51,6 +51,7 @@ parser.add_argument('--network_tag', type=str, default='')
parser.add_argument('--cluster_name', type=str, default='')
parser.add_argument('--service_base_name', type=str, default='')
parser.add_argument('--os_login_enabled', type=str, default='FALSE')
+parser.add_argument('--block_project_ssh_keys', type=str, default='FALSE')
args = parser.parse_args()
@@ -66,7 +67,7 @@ if __name__ == "__main__":
args.secondary_image_name, args.service_account_name, args.instance_class,
args.network_tag, json.loads(args.labels), args.static_ip,
args.primary_disk_size, args.secondary_disk_size, args.gpu_accelerator_type,
- args.gpu_accelerator_count, args.os_login_enabled)
+ args.gpu_accelerator_count, args.os_login_enabled, args.block_project_ssh_keys)
else:
parser.print_help()
sys.exit(2)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
index 9954592..5925480 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
@@ -151,6 +151,9 @@ if __name__ == "__main__":
if os.environ['gcp_os_login_enabled'] != 'FALSE':
notebook_config['gcp_os_login_enabled'] = 'TRUE'
+ if os.environ['gcp_block_project_ssh_keys'] != 'FALSE':
+ notebook_config['gcp_block_project_ssh_keys'] = 'TRUE'
+
notebook_config['gpu_accelerator_type'] = 'None'
notebook_config['gpu_accelerator_count'] = 'None'
@@ -196,7 +199,7 @@ if __name__ == "__main__":
"--ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9} " \
"--secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \
"--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --labels '{17}' " \
- "--service_base_name {18} --os_login_enabled {19}".\
+ "--service_base_name {18} --os_login_enabled {19} --block_project_ssh_keys {20}".\
format(notebook_config['instance_name'], notebook_config['region'], notebook_config['zone'],
notebook_config['vpc_name'], notebook_config['subnet_name'], notebook_config['instance_size'],
notebook_config['ssh_key_path'], notebook_config['initial_user'],
@@ -204,7 +207,8 @@ if __name__ == "__main__":
notebook_config['secondary_image_name'], 'notebook', notebook_config['primary_disk_size'],
notebook_config['secondary_disk_size'], notebook_config['gpu_accelerator_type'],
notebook_config['gpu_accelerator_count'], notebook_config['network_tag'],
- json.dumps(notebook_config['labels']), notebook_config['service_base_name'], notebook_config['gcp_os_login_enabled'])
+ json.dumps(notebook_config['labels']), notebook_config['service_base_name'],
+ notebook_config['gcp_os_login_enabled'], notebook_config['gcp_block_project_ssh_keys'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
index 643354d..a549dee 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
@@ -101,6 +101,9 @@ if __name__ == "__main__":
if os.environ['gcp_os_login_enabled'] != 'FALSE':
data_engine['gcp_os_login_enabled'] = 'TRUE'
+ if os.environ['gcp_block_project_ssh_keys'] != 'FALSE':
+ data_engine['gcp_block_project_ssh_keys'] = 'TRUE'
+
data_engine['cluster_name'] = "{}-{}-{}-de-{}".format(data_engine['service_base_name'],
data_engine['project_name'],
data_engine['endpoint_name'],
@@ -193,14 +196,15 @@ if __name__ == "__main__":
"--ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9} " \
"--secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \
"--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --cluster_name {17} " \
- "--labels '{18}' --service_base_name {19} --os_login_enabled {20}". \
+ "--labels '{18}' --service_base_name {19} --os_login_enabled {20} --block_project_ssh_keys {21}". \
format(data_engine['master_node_name'], data_engine['region'], data_engine['zone'], data_engine['vpc_name'],
data_engine['subnet_name'], data_engine['master_size'], data_engine['ssh_key_path'], initial_user,
data_engine['dataengine_service_account_name'], data_engine['primary_image_name'],
data_engine['secondary_image_name'], 'dataengine', data_engine['primary_disk_size'],
data_engine['secondary_disk_size'], data_engine['gpu_master_accelerator_type'],
data_engine['gpu_master_accelerator_count'], data_engine['network_tag'], data_engine['cluster_name'],
- json.dumps(data_engine['master_labels']), data_engine['service_base_name'], data_engine['gcp_os_login_enabled'])
+ json.dumps(data_engine['master_labels']), data_engine['service_base_name'],
+ data_engine['gcp_os_login_enabled'], data_engine['gcp_block_project_ssh_keys'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
@@ -219,7 +223,7 @@ if __name__ == "__main__":
"--instance_size {5} --ssh_key_path {6} --initial_user {7} --service_account_name {8} " \
"--image_name {9} --secondary_image_name {10} --instance_class {11} --primary_disk_size {12} " \
"--secondary_disk_size {13} --gpu_accelerator_type {14} --gpu_accelerator_count {15} --network_tag {16} --cluster_name {17} " \
- "--labels '{18}' --service_base_name {19} --os_login_enabled {20}". \
+ "--labels '{18}' --service_base_name {19} --os_login_enabled {20} --block_project_ssh_keys {21}". \
format(slave_name, data_engine['region'], data_engine['zone'],
data_engine['vpc_name'], data_engine['subnet_name'], data_engine['slave_size'],
data_engine['ssh_key_path'], initial_user, data_engine['dataengine_service_account_name'],
@@ -228,7 +232,7 @@ if __name__ == "__main__":
data_engine['secondary_disk_size'], data_engine['gpu_slave_accelerator_type'],
data_engine['gpu_slave_accelerator_count'], data_engine['network_tag'],
data_engine['cluster_name'], json.dumps(data_engine['slave_labels']),
- data_engine['service_base_name'], data_engine['gcp_os_login_enabled'])
+ data_engine['service_base_name'], data_engine['gcp_os_login_enabled'], data_engine['gcp_block_project_ssh_keys'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 02e9667..028087d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -503,18 +503,23 @@ if __name__ == "__main__":
if os.environ['gcp_os_login_enabled'] != 'FALSE':
project_conf['gcp_os_login_enabled'] = 'TRUE'
+ if os.environ['gcp_block_project_ssh_keys'] != 'FALSE':
+ project_conf['gcp_block_project_ssh_keys'] = 'TRUE'
+
try:
project_conf['static_ip'] = \
GCPMeta.get_static_address(project_conf['region'], project_conf['static_address_name'])['address']
logging.info('[CREATE EDGE INSTANCE]')
params = "--instance_name {} --region {} --zone {} --vpc_name {} --subnet_name {} --instance_size {} " \
"--ssh_key_path {} --initial_user {} --service_account_name {} --image_name {} --instance_class {} " \
- "--static_ip {} --network_tag {} --labels '{}' --service_base_name {} --os_login_enabled {}".format(
+ "--static_ip {} --network_tag {} --labels '{}' --service_base_name {} --os_login_enabled {} " \
+ "--block_project_ssh_keys {}".format(
project_conf['instance_name'], project_conf['region'], project_conf['zone'], project_conf['vpc_name'],
project_conf['subnet_name'], project_conf['instance_size'], project_conf['ssh_key_path'],
project_conf['initial_user'], project_conf['edge_service_account_name'], project_conf['image_name'],
'edge', project_conf['static_ip'], project_conf['network_tag'],
- json.dumps(project_conf['instance_labels']), project_conf['service_base_name'], project_conf['gcp_os_login_enabled'])
+ json.dumps(project_conf['instance_labels']), project_conf['service_base_name'],
+ project_conf['gcp_os_login_enabled'], project_conf['gcp_block_project_ssh_keys'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
index 9346d15..f4fba87 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
@@ -254,6 +254,9 @@ if __name__ == "__main__":
if os.environ['gcp_os_login_enabled'] != 'FALSE':
ssn_conf['gcp_os_login_enabled'] = 'TRUE'
+ if os.environ['gcp_block_project_ssh_keys'] != 'FALSE':
+ ssn_conf['gcp_block_project_ssh_keys'] = 'TRUE'
+
if os.environ['conf_os_family'] == 'debian':
ssn_conf['initial_user'] = 'ubuntu'
ssn_conf['sudo_group'] = 'sudo'
@@ -268,12 +271,14 @@ if __name__ == "__main__":
params = "--instance_name {0} --region {1} --zone {2} --vpc_name {3} --subnet_name {4} --instance_size {5}"\
" --ssh_key_path {6} --initial_user {7} --service_account_name {8} --image_name {9}"\
" --instance_class {10} --static_ip {11} --network_tag {12} --labels '{13}' " \
- "--primary_disk_size {14} --service_base_name {15} --os_login_enabled {16}".\
+ "--primary_disk_size {14} --service_base_name {15} --os_login_enabled {16} " \
+ "--block_project_ssh_keys {17}".\
format(ssn_conf['instance_name'], ssn_conf['region'], ssn_conf['zone'], ssn_conf['vpc_name'],
ssn_conf['subnet_name'], ssn_conf['instance_size'], ssn_conf['ssh_key_path'],
ssn_conf['initial_user'], ssn_conf['service_account_name'], ssn_conf['image_name'], 'ssn',
ssn_conf['static_ip'], ssn_conf['network_tag'], json.dumps(ssn_conf['instance_labels']), '20',
- ssn_conf['service_base_name'], ssn_conf['gcp_os_login_enabled'])
+ ssn_conf['service_base_name'], ssn_conf['gcp_os_login_enabled'],
+ ssn_conf['gcp_block_project_ssh_keys'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_instance', params), shell=True, check=True)
except:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 04/04: [DATALAB-2674]: added optional parameter to enable bucket versioning
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 986f6086f3c1826aad9142dbb0905070a93cef9b
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Feb 1 16:04:40 2022 +0200
[DATALAB-2674]: added optional parameter to enable bucket versioning
---
infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++
infrastructure-provisioning/src/general/conf/datalab.ini | 4 ++++
.../src/general/lib/gcp/actions_lib.py | 10 ++++++++--
.../src/general/scripts/gcp/common_create_bucket.py | 3 ++-
.../src/general/scripts/gcp/project_prepare.py | 12 ++++++++----
5 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index 40b233d..edafc90 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -263,6 +263,8 @@ def build_parser():
help='TRUE if os login enabled for gcp instances')
gcp_parser.add_argument('--gcp_block_project_ssh_keys', type=str, default='FALSE',
help='TRUE to block project ssh keys for gcp instances')
+ gcp_parser.add_argument('--gcp_bucket_enable_versioning', type=str, default='false',
+ help='TRUE to enable versioning for gcp storage buckets')
gcp_required_args = gcp_parser.add_argument_group('Required arguments')
gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region')
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 73ed7ba..9ae94fa 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -230,6 +230,10 @@ datalake_enable = false
# firewall_rules =
### True if os login is enabled for instances
# os_login_enabled =
+### True if block_project_ssh_keys is enabled for instances
+# block_project_ssh_keys =
+### True if versioning is enabled for buckets
+# bucket_enable_versioning =
### GCP region name for whole DataLab provisioning
region = us-west1
### GCP zone name for whole DataLab provisioning
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 69eb9e7..e92d835 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -210,9 +210,15 @@ class GCPActions:
traceback.print_exc(file=sys.stdout)
- def create_bucket(self, bucket_name):
+ def create_bucket(self, bucket_name, versioning_enabled='false'):
try:
- bucket = self.storage_client.create_bucket(bucket_name)
+ bucket_params = {
+ "name": bucket_name,
+ "versioning": {
+ "enabled": "{}".format(versioning_enabled)
+ }
+ }
+ bucket = self.storage_client.create_bucket(project=self.project, body=bucket_params)
print('Bucket {} created.'.format(bucket.name))
except Exception as err:
logging.info(
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
index 34352e5..0c63b2d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
@@ -31,6 +31,7 @@ from datalab.logger import logging
parser = argparse.ArgumentParser()
parser.add_argument('--bucket_name', type=str, default='')
parser.add_argument('--tags', type=str, default='')
+parser.add_argument('--versioning_enabled', type=str, default='false')
args = parser.parse_args()
if __name__ == "__main__":
@@ -39,7 +40,7 @@ if __name__ == "__main__":
logging.info("REQUESTED BUCKET {} ALREADY EXISTS".format(args.bucket_name))
else:
logging.info("Creating Bucket {}".format(args.bucket_name))
- GCPActions().create_bucket(args.bucket_name)
+ GCPActions().create_bucket(args.bucket_name, args.versioning_enabled)
GCPActions().add_bucket_labels(args.bucket_name, json.loads(args.tags))
else:
parser.print_help()
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 028087d..0ce097b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -121,6 +121,8 @@ if __name__ == "__main__":
else:
project_conf['user_subnets_range'] = ''
+ if os.environ['gcp_bucket_enable_versioning'] != 'false':
+ project_conf['gcp_bucket_enable_versioning'] = 'true'
# FUSE in case of absence of user's key
try:
project_conf['user_key'] = os.environ['key']
@@ -399,8 +401,9 @@ if __name__ == "__main__":
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
"sbn": project_conf['service_base_name'],
"name": project_conf['shared_bucket_name']}
- params = "--bucket_name {} --tags '{}'".format(project_conf['shared_bucket_name'],
- json.dumps(project_conf['shared_bucket_tags']))
+ params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['shared_bucket_name'],
+ json.dumps(project_conf['shared_bucket_tags']),
+ project_conf['gcp_bucket_enable_versioning'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True)
except:
@@ -414,8 +417,9 @@ if __name__ == "__main__":
"sbn": project_conf['service_base_name'],
"project_tag": project_conf['project_tag'],
"name": project_conf['bucket_name']}
- params = "--bucket_name {} --tags '{}'".format(project_conf['bucket_name'],
- json.dumps(project_conf['bucket_tags']))
+ params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['bucket_name'],
+ json.dumps(project_conf['bucket_tags']),
+ project_conf['gcp_bucket_enable_versioning'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org