You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/03/09 12:54:14 UTC
[incubator-datalab] branch DATALAB-2674 created (now 8ac6956)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 8ac6956 [DATALAB-2674]: added lifecycle rules for gcp storage buckets
This branch includes the following new commits:
new 8ac6956 [DATALAB-2674]: added lifecycle rules for gcp storage buckets
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2674]: added lifecycle rules for gcp storage buckets
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 8ac6956aa9049e4f1e3a7198d637c0d93d5011f9
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Mar 9 14:53:29 2022 +0200
[DATALAB-2674]: added lifecycle rules for gcp storage buckets
---
infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++
.../src/general/conf/datalab.ini | 2 ++
.../src/general/lib/gcp/actions_lib.py | 5 ++++-
.../src/general/scripts/gcp/common_create_bucket.py | 4 +++-
.../src/general/scripts/gcp/project_prepare.py | 18 ++++++++++++------
5 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index 8f55428..2de4c5d 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -268,6 +268,8 @@ def build_parser():
gcp_parser.add_argument('--gcp_cmek_resource_name', type=str, default='',
help='customer managed encryption key resource name '
'e.g. projects/{project_name}/locations/{us}/keyRings/{keyring_name}/cryptoKeys/{key_name}')
+ gcp_parser.add_argument('--gcp_storage_lifecycle_rules', type=str, default='',
+ help='storage bucket lifecycle rules')
gcp_parser.add_argument('--gcp_wrapped_csek', type=str, default='',
help='customer supplied encryption key for disk/image encryption in RFC 4648 base64 '
'encoded, RSA-wrapped 2048-bit format as rsaEncryptedKey')
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 336d7e5..fe84416 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -236,6 +236,8 @@ block_project_ssh_keys = FALSE
bucket_enable_versioning = false
### gcp customer managed encryption key to use
# cmek_resource_name =
+### gcp customer managed encryption key to use
+# storage_lifecycle_rules =
### gcp customer supplied wrapped encryption key to use
# wrapped_csek =
### GCP region name for whole DataLab provisioning
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 1ff8a4a..b37cdcd 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -222,7 +222,8 @@ class GCPActions:
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
- def add_bucket_labels_vers_cmek(self, bucket_name, tags, versioning_enabled='false', cmek_resource_name=''):
+ def add_bucket_labels_vers_cmek(self, bucket_name, tags, versioning_enabled='false', cmek_resource_name='',
+ lifecycle_rules=''):
try:
bucket = self.storage_client.get_bucket(bucket_name)
labels = bucket.labels
@@ -231,6 +232,8 @@ class GCPActions:
bucket.versioning = {"enabled": versioning_enabled}
if cmek_resource_name != '':
bucket.encryption = {"defaultKmsKeyName": cmek_resource_name}
+ if lifecycle_rules != '':
+ bucket.lifecycle = lifecycle_rules
bucket.patch()
print('Updated labels on {}.'.format(bucket_name))
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
index 5a49201..389ca10 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
@@ -33,6 +33,7 @@ parser.add_argument('--bucket_name', type=str, default='')
parser.add_argument('--tags', type=str, default='')
parser.add_argument('--versioning_enabled', type=str, default='false')
parser.add_argument('--cmek_resource_name', type=str, default='')
+parser.add_argument('--lifecycle_rules', type=str, default='')
args = parser.parse_args()
if __name__ == "__main__":
@@ -42,7 +43,8 @@ if __name__ == "__main__":
else:
logging.info("Creating Bucket {}".format(args.bucket_name))
GCPActions().create_bucket(args.bucket_name)
- GCPActions().add_bucket_labels_vers_cmek(args.bucket_name, json.loads(args.tags), args.versioning_enabled, args.cmek_resource_name)
+ GCPActions().add_bucket_labels_vers_cmek(args.bucket_name, json.loads(args.tags), args.versioning_enabled,
+ args.cmek_resource_name, json.loads(args.lifecycle_rules))
else:
parser.print_help()
sys.exit(2)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 15a7270..a8da0ec 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -126,6 +126,12 @@ if __name__ == "__main__":
project_conf['gcp_cmek_resource_name'] = os.environ['gcp_cmek_resource_name']
else:
project_conf['gcp_cmek_resource_name'] = ''
+
+ if 'gcp_storage_lifecycle_rules' in os.environ:
+ project_conf['gcp_storage_lifecycle_rules'] = os.environ['gcp_storage_lifecycle_rules']
+ else:
+ project_conf['gcp_storage_lifecycle_rules'] = ''
+
# FUSE in case of absence of user's key
try:
project_conf['user_key'] = os.environ['key']
@@ -404,9 +410,9 @@ if __name__ == "__main__":
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
"sbn": project_conf['service_base_name'],
"name": project_conf['shared_bucket_name']}
- params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['shared_bucket_name'],
- json.dumps(project_conf['shared_bucket_tags']),
- project_conf['gcp_bucket_enable_versioning'])
+ params = "--bucket_name {} --tags '{}' --versioning_enabled {} --lifecycle_rules '{}'".format(
+ project_conf['shared_bucket_name'], json.dumps(project_conf['shared_bucket_tags']),
+ project_conf['gcp_bucket_enable_versioning'], json.dumps(project_conf['gcp_storage_lifecycle_rules']))
if project_conf['gcp_cmek_resource_name'] != '':
params = '{} --cmek_resource_name {}'.format(params, project_conf['gcp_cmek_resource_name'])
@@ -424,9 +430,9 @@ if __name__ == "__main__":
"sbn": project_conf['service_base_name'],
"project_tag": project_conf['project_tag'],
"name": project_conf['bucket_name']}
- params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['bucket_name'],
- json.dumps(project_conf['bucket_tags']),
- project_conf['gcp_bucket_enable_versioning'])
+ params = "--bucket_name {} --tags '{}' --versioning_enabled {} --lifecycle_rules '{}'".format(
+ project_conf['bucket_name'], json.dumps(project_conf['bucket_tags']),
+ project_conf['gcp_bucket_enable_versioning'], json.dumps(project_conf['gcp_storage_lifecycle_rules']))
if project_conf['gcp_cmek_resource_name'] != '':
params = '{} --cmek_resource_name {}'.format(params, project_conf['gcp_cmek_resource_name'])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org