You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/02/04 10:36:22 UTC
[incubator-datalab] 04/04: [DATALAB-2674]: added optional parameter to enable bucket versioning
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 986f6086f3c1826aad9142dbb0905070a93cef9b
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Feb 1 16:04:40 2022 +0200
[DATALAB-2674]: added optional parameter to enable bucket versioning
---
infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++
infrastructure-provisioning/src/general/conf/datalab.ini | 4 ++++
.../src/general/lib/gcp/actions_lib.py | 10 ++++++++--
.../src/general/scripts/gcp/common_create_bucket.py | 3 ++-
.../src/general/scripts/gcp/project_prepare.py | 12 ++++++++----
5 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index 40b233d..edafc90 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -263,6 +263,8 @@ def build_parser():
help='TRUE if os login enabled for gcp instances')
gcp_parser.add_argument('--gcp_block_project_ssh_keys', type=str, default='FALSE',
help='TRUE to block project ssh keys for gcp instances')
+ gcp_parser.add_argument('--gcp_bucket_enable_versioning', type=str, default='false',
+ help='TRUE to enable versioning for gcp storage buckets')
gcp_required_args = gcp_parser.add_argument_group('Required arguments')
gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region')
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 73ed7ba..9ae94fa 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -230,6 +230,10 @@ datalake_enable = false
# firewall_rules =
### True if os login is enabled for instances
# os_login_enabled =
+### True if block_project_ssh_keys is enabled for instances
+# block_project_ssh_keys =
+### True if versioning is enabled for buckets
+# bucket_enable_versioning =
### GCP region name for whole DataLab provisioning
region = us-west1
### GCP zone name for whole DataLab provisioning
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 69eb9e7..e92d835 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -210,9 +210,15 @@ class GCPActions:
traceback.print_exc(file=sys.stdout)
- def create_bucket(self, bucket_name):
+ def create_bucket(self, bucket_name, versioning_enabled='false'):
try:
- bucket = self.storage_client.create_bucket(bucket_name)
+ bucket_params = {
+ "name": bucket_name,
+ "versioning": {
+ "enabled": "{}".format(versioning_enabled)
+ }
+ }
+ bucket = self.storage_client.create_bucket(project=self.project, body=bucket_params)
print('Bucket {} created.'.format(bucket.name))
except Exception as err:
logging.info(
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
index 34352e5..0c63b2d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
@@ -31,6 +31,7 @@ from datalab.logger import logging
parser = argparse.ArgumentParser()
parser.add_argument('--bucket_name', type=str, default='')
parser.add_argument('--tags', type=str, default='')
+parser.add_argument('--versioning_enabled', type=str, default='false')
args = parser.parse_args()
if __name__ == "__main__":
@@ -39,7 +40,7 @@ if __name__ == "__main__":
logging.info("REQUESTED BUCKET {} ALREADY EXISTS".format(args.bucket_name))
else:
logging.info("Creating Bucket {}".format(args.bucket_name))
- GCPActions().create_bucket(args.bucket_name)
+ GCPActions().create_bucket(args.bucket_name, args.versioning_enabled)
GCPActions().add_bucket_labels(args.bucket_name, json.loads(args.tags))
else:
parser.print_help()
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 028087d..0ce097b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -121,6 +121,8 @@ if __name__ == "__main__":
else:
project_conf['user_subnets_range'] = ''
+ if os.environ['gcp_bucket_enable_versioning'] != 'false':
+ project_conf['gcp_bucket_enable_versioning'] = 'true'
# FUSE in case of absence of user's key
try:
project_conf['user_key'] = os.environ['key']
@@ -399,8 +401,9 @@ if __name__ == "__main__":
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
"sbn": project_conf['service_base_name'],
"name": project_conf['shared_bucket_name']}
- params = "--bucket_name {} --tags '{}'".format(project_conf['shared_bucket_name'],
- json.dumps(project_conf['shared_bucket_tags']))
+ params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['shared_bucket_name'],
+ json.dumps(project_conf['shared_bucket_tags']),
+ project_conf['gcp_bucket_enable_versioning'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True)
except:
@@ -414,8 +417,9 @@ if __name__ == "__main__":
"sbn": project_conf['service_base_name'],
"project_tag": project_conf['project_tag'],
"name": project_conf['bucket_name']}
- params = "--bucket_name {} --tags '{}'".format(project_conf['bucket_name'],
- json.dumps(project_conf['bucket_tags']))
+ params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['bucket_name'],
+ json.dumps(project_conf['bucket_tags']),
+ project_conf['gcp_bucket_enable_versioning'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org