You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by ol...@apache.org on 2022/02/28 13:56:05 UTC
[incubator-datalab] branch gcp-jupyter-highgpu-template-1 updated: template 1
This is an automated email from the ASF dual-hosted git repository.
olehmykolaishyn pushed a commit to branch gcp-jupyter-highgpu-template-1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/gcp-jupyter-highgpu-template-1 by this push:
new 694be17 template 1
694be17 is described below
commit 694be17dd157bf8c631818d7b0a63b710917d469
Author: oleh_mykolaishyn <ow...@gmail.com>
AuthorDate: Mon Feb 28 15:55:47 2022 +0200
template 1
---
.../src/general/conf/datalab.ini | 2 +-
.../src/general/files/gcp/jupyter-gpu_Dockerfile | 2 +-
.../src/general/scripts/gcp/ssn_configure.py | 1 +
.../scripts/configure_jupyter-gpu_node.py | 8 ++++++++
.../src/ssn/files/gcp/mongo_roles.json | 20 ++++++++++++++++++
.../terraform/bin/deploy/endpoint_fab.py | 2 +-
.../terraform/gcp/endpoint/provisioning.py | 2 ++
.../core/commands/CommandExecutorMock.java | 1 +
.../backendapi/domain/NotebookTemplate.java | 1 +
.../epam/datalab/backendapi/util/BillingUtils.java | 2 +-
.../src/main/resources/mongo/gcp/mongo_roles.json | 24 ++++++++++++++++++++++
11 files changed, 61 insertions(+), 4 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 1de4068..5970912 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -391,7 +391,7 @@ slave_instance_spot_pct_price = 70
### Count of slave nodes for Data Engine
# instance_count =
### Type of notebooks for creating Data Engine from notebook images
-image_notebooks = jupyter,jupyterlab,rstudio,zeppelin,tensor,tensor-rstudio,deeplearning
+image_notebooks = jupyter,jupyter-gpu,jupyterlab,rstudio,zeppelin,tensor,tensor-rstudio,deeplearning
### Persent of RAM allocated for an operating system
os_memory = 75
### Explicit allocation RAM for an operating system
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu_Dockerfile
index 4694dd7..87c4114 100644
--- a/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyter-gpu_Dockerfile
@@ -24,7 +24,7 @@ FROM docker.datalab-base:latest
ARG OS
-COPY jupyter/ /root/
+COPY jupyter-gpu/ /root/
COPY general/scripts/os/* /root/scripts/
COPY general/scripts/gcp/jupyter-gpu_* /root/scripts/
COPY general/lib/os/${OS}/notebook_lib.py /usr/lib/python3.8/datalab/notebook_lib.py
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index d55fce0..f96c911 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -202,6 +202,7 @@ if __name__ == "__main__":
{"name": "project", "tag": "latest"},
{"name": "edge", "tag": "latest"},
{"name": "jupyter", "tag": "latest"},
+ {"name": "jupyter-gpu", "tag": "latest"},
{"name": "jupyterlab", "tag": "latest"},
{"name": "rstudio", "tag": "latest"},
{"name": "zeppelin", "tag": "latest"},
diff --git a/infrastructure-provisioning/src/jupyter-gpu/scripts/configure_jupyter-gpu_node.py b/infrastructure-provisioning/src/jupyter-gpu/scripts/configure_jupyter-gpu_node.py
index 3369ab2..bc94dc2 100644
--- a/infrastructure-provisioning/src/jupyter-gpu/scripts/configure_jupyter-gpu_node.py
+++ b/infrastructure-provisioning/src/jupyter-gpu/scripts/configure_jupyter-gpu_node.py
@@ -65,6 +65,7 @@ local_spark_path = '/opt/spark/'
toree_link = 'https://dist.apache.org/repos/dist/dev/incubator/toree/0.5.0-incubating-rc1/toree-pip/toree-0.5.0.tar.gz'
r_libs = ['R6', 'pbdZMQ={}'.format(os.environ['notebook_pbdzmq_version']), 'RCurl', 'reshape2', 'caTools={}'.format(os.environ['notebook_catools_version']), 'rJava', 'ggplot2']
gitlab_certfile = os.environ['conf_gitlab_certfile']
+venv_libs = 'numpy scipy pandas scikit-learn git transformers==4.4.2 gensim==4.0.1 tokenizers==0.10.1 python-levenshtein==0.12.2'
##############
@@ -147,6 +148,13 @@ if __name__ == "__main__":
ensure_sbt(args.os_user)
print("Install Breeze")
add_breeze_library_local(args.os_user)
+ if os.environ['conf_cloud_provider'] == 'gcp':
+ print('Installing Pytorch')
+ ensure_pytorch(args.os_user)
+
+ # INSTALL PIP PACKAGES
+ print("Install python venv required libs")
+ ensure_venv_libs(args.os_user, venv_libs)
#POST INSTALLATION PROCESS
print("Updating pyOpenSSL library")
diff --git a/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json b/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json
index badb966..b109f48 100644
--- a/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json
+++ b/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json
@@ -90,6 +90,16 @@
]
},
{
+ "_id": "nbCreateJupyterGpu",
+ "description": "Create Notebook Jupyter Gpu",
+ "exploratories": [
+ "docker.datalab-jupyter-gpu"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
"_id": "nbCreateJupyterLab",
"description": "Create Notebook JupyterLab",
"exploratories": [
@@ -240,6 +250,16 @@
]
},
{
+ "_id": "compShapes_a2-highgpu-1g_fetching",
+ "description": "Use a2-highgpu-1g instance shape for cluster",
+ "computational_shapes": [
+ "a2-highgpu-1g"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
"_id": "nbBillingReportFull",
"description": "View full billing report for all users",
"pages": [
diff --git a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
index e5bd059..454f348 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
+++ b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
@@ -640,7 +640,7 @@ def pull_docker_images():
list_images = {
'aws': ['base', 'edge', 'project', 'jupyter', 'rstudio', 'zeppelin', 'tensor', 'tensor-rstudio',
'deeplearning', 'jupyterlab', 'dataengine-service', 'dataengine'],
- 'gcp': ['base', 'edge', 'project', 'jupyter', 'rstudio', 'zeppelin', 'tensor', 'tensor-rstudio',
+ 'gcp': ['base', 'edge', 'project', 'jupyter', 'jupyter-gpu', 'rstudio', 'zeppelin', 'tensor', 'tensor-rstudio',
'deeplearning', 'superset', 'jupyterlab', 'dataengine-service', 'dataengine'],
'azure': ['base', 'edge', 'project', 'jupyter', 'rstudio', 'zeppelin', 'tensor', 'deeplearning',
'dataengine']
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
index ed83bb6..af3befd 100644
--- a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
@@ -402,6 +402,8 @@ def pull_docker_images():
.format(args.repository_address, args.repository_port, args.cloud_provider))
conn.sudo('docker pull {}:{}/docker.datalab-jupyter-{}'
.format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.datalab-jupyter-gpu-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
conn.sudo('docker pull {}:{}/docker.datalab-rstudio-{}'
.format(args.repository_address, args.repository_port, args.cloud_provider))
conn.sudo('docker pull {}:{}/docker.datalab-zeppelin-{}'
diff --git a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/commands/CommandExecutorMock.java b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/commands/CommandExecutorMock.java
index 33bd495..a058060 100644
--- a/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/commands/CommandExecutorMock.java
+++ b/services/provisioning-service/src/main/java/com/epam/datalab/backendapi/core/commands/CommandExecutorMock.java
@@ -78,6 +78,7 @@ public class CommandExecutorMock implements ICommandExecutor {
List<String> list = Lists.newArrayList(
"docker.datalab-deeplearning:latest",
"docker.datalab-jupyter:latest",
+ "docker.datalab-jupyter-gpu:latest",
"docker.datalab-jupyterlab:latest",
"docker.datalab-superset:latest",
"docker.datalab-rstudio:latest",
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java
index ad5e954..f3abcc5 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java
@@ -26,6 +26,7 @@ import lombok.Getter;
@AllArgsConstructor
public enum NotebookTemplate {
JUPYTER("Jupyter notebook 6.1.6"),
+ JUPYTER_GPU("Jupyter with GPU notebook 6.1.6"),
JUPYTER_LAB("JupyterLab 0.35.6"),
ZEPPELIN("Apache Zeppelin 0.9.1"),
DEEP_LEARNING("Deep Learning 2.4"),
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
index bf425ea..58d2488 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/util/BillingUtils.java
@@ -39,7 +39,7 @@ import static com.epam.datalab.dto.billing.BillingResourceType.*;
@Slf4j
public class BillingUtils {
- private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "deeplearning"};
+ private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "jupyter-gpu","deeplearning"};
private static final String[] BILLING_FILTERED_REPORT_HEADERS = {"DataLab ID", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"};
private static final String[] COMPLETE_REPORT_REPORT_HEADERS = {"DataLab ID", "User", "Project", "DataLab Resource Type", "Status", "Shape", "Product", "Cost"};
diff --git a/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json b/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json
index e75c29c..f3bd5eb 100644
--- a/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json
+++ b/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json
@@ -108,6 +108,18 @@
]
},
{
+ "_id": "nbCreateJupyterGpu",
+ "description": "Create Notebook Jupyter Gpu",
+ "type": "NOTEBOOK",
+ "cloud": "GCP",
+ "exploratories": [
+ "docker.datalab-jupyter-gpu"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
+ },
+ {
"_id": "nbCreateJupyterLab",
"description": "Create Notebook JupyterLab",
"type": "NOTEBOOK",
@@ -286,5 +298,17 @@
"groups": [
"$anyuser"
]
+ },
+ {
+ "_id": "compShapes_a2-highgpu-1g_fetching",
+ "description": "Use a2-highgpu-1g instance shape for cluster",
+ "type": "COMPUTATIONAL_SHAPE",
+ "cloud": "GCP",
+ "computational_shapes": [
+ "a2-highgpu-1g"
+ ],
+ "groups": [
+ "$anyuser"
+ ]
}
]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org