You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/02/09 10:35:18 UTC
[incubator-datalab] branch DATALAB-2674 updated: [DATALAB-2674]: added disk and image encryption with wrapped csek
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2674 by this push:
new 9f31a11 [DATALAB-2674]: added disk and image encryption with wrapped csek
9f31a11 is described below
commit 9f31a11051ca95f3d0bb45a31780fa6e29f59f37
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Feb 9 12:35:06 2022 +0200
[DATALAB-2674]: added disk and image encryption with wrapped csek
---
.../src/general/scripts/gcp/common_create_notebook_image.py | 7 ++++++-
.../src/general/scripts/gcp/common_prepare_notebook.py | 7 ++++++-
.../src/general/scripts/gcp/dataengine_prepare.py | 5 ++++-
.../src/general/scripts/gcp/deeplearning_configure.py | 7 ++++++-
.../src/general/scripts/gcp/jupyter_configure.py | 6 +++++-
.../src/general/scripts/gcp/jupyterlab_configure.py | 6 +++++-
.../src/general/scripts/gcp/project_prepare.py | 5 ++++-
.../src/general/scripts/gcp/rstudio_configure.py | 6 +++++-
infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py | 6 +++++-
.../src/general/scripts/gcp/superset_configure.py | 6 +++++-
.../src/general/scripts/gcp/tensor-rstudio_configure.py | 6 +++++-
.../src/general/scripts/gcp/tensor_configure.py | 6 +++++-
.../src/general/scripts/gcp/zeppelin_configure.py | 6 +++++-
13 files changed, 66 insertions(+), 13 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_create_notebook_image.py b/infrastructure-provisioning/src/general/scripts/gcp/common_create_notebook_image.py
index 25c8a54..35237ae 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_create_notebook_image.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_create_notebook_image.py
@@ -64,6 +64,11 @@ if __name__ == "__main__":
image_conf['endpoint_name'],
image_conf['exploratory_name'])
+ if "gcp_wrapped_csek" in os.environ:
+ image_conf['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ image_conf['gcp_wrapped_csek'] = ''
+
image_conf['zone'] = os.environ['gcp_zone']
logging.info('[CREATING IMAGE]')
primary_image_id = GCPMeta.get_image_by_name(image_conf['expected_primary_image_name'])
@@ -71,7 +76,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
image_conf['expected_primary_image_name'], image_conf['expected_secondary_image_name'],
image_conf['instance_name'], image_conf['zone'], image_conf['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ image_conf['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
index 6d8e3d3..814a227 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
@@ -150,7 +150,12 @@ if __name__ == "__main__":
notebook_config['gcp_os_login_enabled'] = os.environ['gcp_os_login_enabled']
notebook_config['gcp_block_project_ssh_keys'] = os.environ['gcp_block_project_ssh_keys']
- notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
+
notebook_config['gpu_accelerator_type'] = 'None'
notebook_config['gpu_accelerator_count'] = 'None'
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
index d2cd931..84893b4 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
@@ -100,7 +100,10 @@ if __name__ == "__main__":
data_engine['gcp_os_login_enabled'] = os.environ['gcp_os_login_enabled']
data_engine['gcp_block_project_ssh_keys'] = os.environ['gcp_block_project_ssh_keys']
- data_engine['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ if "gcp_wrapped_csek" in os.environ:
+ data_engine['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ data_engine['gcp_wrapped_csek'] = ''
data_engine['cluster_name'] = "{}-{}-{}-de-{}".format(data_engine['service_base_name'],
data_engine['project_name'],
data_engine['endpoint_name'],
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
index 6c3258d..f5c294b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
@@ -85,6 +85,11 @@ if __name__ == "__main__":
notebook_config['datalab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
+
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
@@ -209,7 +214,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ notebook_config['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index 05d7c51..caa17e1 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -85,6 +85,10 @@ if __name__ == "__main__":
notebook_config['datalab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
@@ -211,7 +215,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ notebook_config['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
index d85930d..2baa8ee 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyterlab_configure.py
@@ -85,6 +85,10 @@ if __name__ == "__main__":
notebook_config['datalab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
@@ -209,7 +213,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ notebook_config['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 446c8e6..1da4145 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -513,7 +513,10 @@ if __name__ == "__main__":
project_conf['gcp_os_login_enabled'] = os.environ['gcp_os_login_enabled']
project_conf['gcp_block_project_ssh_keys'] = os.environ['gcp_block_project_ssh_keys']
- project_conf['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ if "gcp_wrapped_csek" in os.environ:
+ project_conf['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ project_conf['gcp_wrapped_csek'] = ''
try:
project_conf['static_ip'] = \
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
index f1ae637..c5969ef 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_configure.py
@@ -88,6 +88,10 @@ if __name__ == "__main__":
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
notebook_config['ip_address'] = GCPMeta.get_private_ip_address(notebook_config['instance_name'])
notebook_config['rstudio_pass'] = datalab.fab.id_generator()
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
@@ -213,7 +217,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ notebook_config['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
index 54fddef..78d7621 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_prepare.py
@@ -73,7 +73,11 @@ if __name__ == "__main__":
ssn_conf['allowed_ip_cidr'] = os.environ['conf_allowed_ip_cidr']
ssn_conf['gcp_os_login_enabled'] = os.environ['gcp_os_login_enabled']
ssn_conf['gcp_block_project_ssh_keys'] = os.environ['gcp_block_project_ssh_keys']
- ssn_conf['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+
+ if "gcp_wrapped_csek" in os.environ:
+ ssn_conf['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ ssn_conf['gcp_wrapped_csek'] = ''
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary.", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
index 8680bee..44295da 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/superset_configure.py
@@ -87,6 +87,10 @@ if __name__ == "__main__":
notebook_config['datalab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
@@ -255,7 +259,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ notebook_config['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
index d29af7b..423d444 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/tensor-rstudio_configure.py
@@ -92,6 +92,10 @@ if __name__ == "__main__":
notebook_config['zone'] = os.environ['gcp_zone']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
notebook_config['rstudio_pass'] = datalab.fab.id_generator()
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
@@ -215,7 +219,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ notebook_config['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
index 4c3dfec..9a4557b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/tensor_configure.py
@@ -86,6 +86,10 @@ if __name__ == "__main__":
notebook_config['datalab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
@@ -220,7 +224,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ notebook_config['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
index 5bdc344..00422ea 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_configure.py
@@ -86,6 +86,10 @@ if __name__ == "__main__":
notebook_config['datalab_ssh_user'] = os.environ['conf_os_user']
notebook_config['zone'] = os.environ['gcp_zone']
notebook_config['shared_image_enabled'] = os.environ['conf_shared_image_enabled']
+ if "gcp_wrapped_csek" in os.environ:
+ notebook_config['gcp_wrapped_csek'] = os.environ['gcp_wrapped_csek']
+ else:
+ notebook_config['gcp_wrapped_csek'] = ''
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
@@ -220,7 +224,7 @@ if __name__ == "__main__":
image_id_list = GCPActions.create_image_from_instance_disks(
notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- os.environ['gcp_wrapped_csek'])
+ notebook_config['gcp_wrapped_csek'])
if image_id_list and image_id_list[0] != '':
logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
else:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org