You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/02/07 13:50:06 UTC
[incubator-datalab] 01/01: [DATALAB-2674]: cmek argument for gcp buckets
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5122ddefe0ba3ca94a9dfcd27915f8c40b50a1c0
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Mon Feb 7 15:49:53 2022 +0200
[DATALAB-2674]: cmek argument for gcp buckets
---
infrastructure-provisioning/scripts/deploy_datalab.py | 3 +++
infrastructure-provisioning/src/general/conf/datalab.ini | 2 ++
infrastructure-provisioning/src/general/lib/gcp/actions_lib.py | 6 ++++--
.../src/general/scripts/gcp/common_create_bucket.py | 3 ++-
.../src/general/scripts/gcp/project_prepare.py | 8 ++++++++
5 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index 67db8da..f4587a8 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -265,6 +265,9 @@ def build_parser():
help='"TRUE" to block project ssh keys for gcp instances')
gcp_parser.add_argument('--gcp_bucket_enable_versioning', type=str, default='false',
help='"true" to enable versioning for gcp storage buckets')
+ gcp_parser.add_argument('--gcp_cmek_resource_name', type=str, default='',
+ help='customer managed encryption key resource name '
+ 'e.g. projects/{project_name}/locations/{us}/keyRings/{keyring_name}/cryptoKeys/{key_name}')
gcp_required_args = gcp_parser.add_argument_group('Required arguments')
gcp_required_args.add_argument('--gcp_region', type=str, required=True, help='GCP region')
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index c5bac75..b495302 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -234,6 +234,8 @@ os_login_enabled = FALSE
block_project_ssh_keys = FALSE
### True if versioning is enabled for buckets
bucket_enable_versioning = false
+### gcp customer managed encryption key to use
+# cmek_resource_name =
### GCP region name for whole DataLab provisioning
region = us-west1
### GCP zone name for whole DataLab provisioning
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index e92d835..531d6fc 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -210,14 +210,16 @@ class GCPActions:
traceback.print_exc(file=sys.stdout)
- def create_bucket(self, bucket_name, versioning_enabled='false'):
+ def create_bucket(self, bucket_name, versioning_enabled='false', cmek_resource_name=''):
try:
bucket_params = {
"name": bucket_name,
"versioning": {
"enabled": "{}".format(versioning_enabled)
- }
+ }
}
+ if cmek_resource_name != '':
+ bucket_params["encryption"] = {"defaultKmsKeyName": cmek_resource_name}
bucket = self.storage_client.create_bucket(project=self.project, body=bucket_params)
print('Bucket {} created.'.format(bucket.name))
except Exception as err:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
index 0c63b2d..0291dbb 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_bucket.py
@@ -32,6 +32,7 @@ parser = argparse.ArgumentParser()
parser.add_argument('--bucket_name', type=str, default='')
parser.add_argument('--tags', type=str, default='')
parser.add_argument('--versioning_enabled', type=str, default='false')
+parser.add_argument('--cmek_resource_name', type=str, default='')
args = parser.parse_args()
if __name__ == "__main__":
@@ -40,7 +41,7 @@ if __name__ == "__main__":
logging.info("REQUESTED BUCKET {} ALREADY EXISTS".format(args.bucket_name))
else:
logging.info("Creating Bucket {}".format(args.bucket_name))
- GCPActions().create_bucket(args.bucket_name, args.versioning_enabled)
+ GCPActions().create_bucket(args.bucket_name, args.versioning_enabled, args.cmek_resource_name)
GCPActions().add_bucket_labels(args.bucket_name, json.loads(args.tags))
else:
parser.print_help()
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 1ebbcae..56591cf 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -122,6 +122,7 @@ if __name__ == "__main__":
project_conf['user_subnets_range'] = ''
project_conf['gcp_bucket_enable_versioning'] = os.environ['gcp_bucket_enable_versioning']
+ project_conf['gcp_cmek_resource_name'] = os.environ['gcp_cmek_resource_name']
# FUSE in case of absence of user's key
try:
project_conf['user_key'] = os.environ['key']
@@ -403,6 +404,10 @@ if __name__ == "__main__":
params = "--bucket_name {} --tags '{}' --versioning_enabled {}".format(project_conf['shared_bucket_name'],
json.dumps(project_conf['shared_bucket_tags']),
project_conf['gcp_bucket_enable_versioning'])
+
+ if project_conf['gcp_cmek_resource_name'] != '':
+ params = '{} --cmek_resource_name {}'.format(params, project_conf['gcp_cmek_resource_name'])
+
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True)
except:
@@ -420,6 +425,9 @@ if __name__ == "__main__":
json.dumps(project_conf['bucket_tags']),
project_conf['gcp_bucket_enable_versioning'])
+ if project_conf['gcp_cmek_resource_name'] != '':
+ params = '{} --cmek_resource_name {}'.format(params, project_conf['gcp_cmek_resource_name'])
+
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True)
except:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org