You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by ol...@apache.org on 2022/02/22 09:40:34 UTC
[incubator-datalab] branch security/DLAB-2882 updated: aws bucket versioning with parsing
This is an automated email from the ASF dual-hosted git repository.
olehmykolaishyn pushed a commit to branch security/DLAB-2882
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/security/DLAB-2882 by this push:
new 54bebed aws bucket versioning with parsing
54bebed is described below
commit 54bebed371b64aa38f488f8642fa7107bb7864a2
Author: oleh_mykolaishyn <ow...@gmail.com>
AuthorDate: Tue Feb 22 11:40:13 2022 +0200
aws bucket versioning with parsing
---
infrastructure-provisioning/scripts/deploy_datalab.py | 2 ++
.../src/general/lib/aws/actions_lib.py | 7 ++++---
.../src/general/scripts/aws/common_create_bucket.py | 4 +++-
.../src/general/scripts/aws/project_prepare.py | 11 +++++++----
.../src/general/templates/aws/edge_s3_policy.json | 6 ++++--
5 files changed, 20 insertions(+), 10 deletions(-)
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index 8f55428..c86aa5e 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -212,6 +212,8 @@ def build_parser():
help='Permission boundary to be attached to new roles')
aws_parser.add_argument('--aws_ssn_instance_role', type=str, default='',
help='Role to be attached to SSN instance')
+ aws_parser.add_argument('--aws_bucket_versioning_enabled', type=str, default='true', choices=BOOL_CHOICES_LIST,
+ help='Versioning for S3 bucket (valid choices: %s)' % BOOL_CHOICES_LIST)
aws_required_args = aws_parser.add_argument_group('Required arguments')
aws_required_args.add_argument('--aws_region', type=str, required=True, help='AWS region')
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 03ed634..010ff23 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -65,15 +65,16 @@ def put_to_bucket(bucket_name, local_file, destination_file):
return False
-def create_s3_bucket(bucket_name, bucket_tags, region, bucket_name_tag):
+def create_s3_bucket(bucket_name, bucket_tags, region, bucket_name_tag, bucket_versioning_enabled):
try:
s3 = boto3.resource('s3', config=botoConfig(signature_version='s3v4'))
if region == "us-east-1":
bucket = s3.create_bucket(Bucket=bucket_name)
else:
bucket = s3.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': region})
- bucket_versioning = s3.BucketVersioning(bucket_name)
- bucket_versioning.enable()
+ if bucket_versioning_enabled == "true":
+ bucket_versioning = s3.BucketVersioning(bucket_name)
+ bucket_versioning.enable()
boto3.client('s3', config=botoConfig(signature_version='s3v4')).put_bucket_encryption(
Bucket=bucket_name, ServerSideEncryptionConfiguration={
'Rules': [
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py b/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
index b34a83a..16f5cd0 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_create_bucket.py
@@ -32,6 +32,7 @@ parser.add_argument('--bucket_name', type=str, default='')
parser.add_argument('--bucket_tags', type=str, default='')
parser.add_argument('--region', type=str, default='')
parser.add_argument('--bucket_name_tag', type=str, default='')
+parser.add_argument('--bucket_versioning_enabled', type=str, default='')
args = parser.parse_args()
if __name__ == "__main__":
@@ -40,7 +41,8 @@ if __name__ == "__main__":
bucket = get_bucket_by_name(args.bucket_name)
if bucket == '':
logging.info("Creating bucket {0} with tags {1}.".format(args.bucket_name, args.bucket_tags))
- bucket = create_s3_bucket(args.bucket_name, args.bucket_tags, args.region, args.bucket_name_tag)
+ bucket = create_s3_bucket(args.bucket_name, args.bucket_tags, args.region, args.bucket_name_tag,
+ args.bucket_versioning_enabled)
else:
logging.info("REQUESTED BUCKET ALREADY EXISTS")
logging.info("BUCKET_NAME {}".format(bucket))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
index aa271cd..dd2518e 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/project_prepare.py
@@ -62,6 +62,9 @@ if __name__ == "__main__":
project_conf['project_name'],
project_conf['endpoint_name'])
project_conf['bucket_name'] = project_conf['bucket_name_tag'].lower().replace('_', '-')
+ #
+ project_conf['bucket_versioning_enabled'] = os.environ['aws_bucket_versioning_enabled']
+ #
project_conf['shared_bucket_name_tag'] = '{0}-{1}-shared-bucket'.format(
project_conf['service_base_name'], project_conf['endpoint_tag'])
project_conf['shared_bucket_name'] = project_conf['shared_bucket_name_tag'].lower().replace('_', '-')
@@ -556,9 +559,9 @@ if __name__ == "__main__":
project_conf['endpoint_tag'], os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'],
project_conf['tag_name'], project_conf['shared_bucket_name'],
project_conf['bucket_additional_tags']).replace(';', ',')
- params = "--bucket_name {} --bucket_tags {} --region {} --bucket_name_tag {}". \
+ params = "--bucket_name {} --bucket_tags {} --region {} --bucket_name_tag {} --bucket_versioning_enabled {}". \
format(project_conf['shared_bucket_name'], project_conf['shared_bucket_tags'], project_conf['region'],
- project_conf['shared_bucket_name_tag'])
+ project_conf['shared_bucket_name_tag'], project_conf['bucket_versioning_enabled'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True)
except:
@@ -568,9 +571,9 @@ if __name__ == "__main__":
project_conf['endpoint_tag'], os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'],
project_conf['project_tag'], project_conf['tag_name'], project_conf['bucket_name'],
project_conf['bucket_additional_tags']).replace(';', ',')
- params = "--bucket_name {} --bucket_tags {} --region {} --bucket_name_tag {}" \
+ params = "--bucket_name {} --bucket_tags {} --region {} --bucket_name_tag {} --bucket_versioning_enabled {}" \
.format(project_conf['bucket_name'], project_conf['bucket_tags'], project_conf['region'],
- project_conf['bucket_name_tag'])
+ project_conf['bucket_name_tag'], project_conf['bucket_versioning_enabled'])
try:
subprocess.run("~/scripts/{}.py {}".format('common_create_bucket', params), shell=True, check=True)
except:
diff --git a/infrastructure-provisioning/src/general/templates/aws/edge_s3_policy.json b/infrastructure-provisioning/src/general/templates/aws/edge_s3_policy.json
index dac85ad..3fc4b81 100644
--- a/infrastructure-provisioning/src/general/templates/aws/edge_s3_policy.json
+++ b/infrastructure-provisioning/src/general/templates/aws/edge_s3_policy.json
@@ -12,7 +12,8 @@
"s3:ListBucket",
"s3:GetBucketLocation",
"s3:PutBucketPolicy",
- "s3:PutEncryptionConfiguration"
+ "s3:PutEncryptionConfiguration",
+ "s3:PutBucketVersioning"
],
"Resource": [
"arn:aws:s3:::SSN_BUCK",
@@ -35,7 +36,8 @@
"s3:Put*",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
- "s3:AbortMultipartUpload"
+ "s3:AbortMultipartUpload",
+ "s3:PutBucketVersioning"
],
"Resource": [
"arn:aws:s3:::BUCKET_NAME/*",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org