You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/11/02 08:47:01 UTC
[incubator-datalab] branch epm-v2.5.2.1 updated (dcfd333cd -> 25a448aa3)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
from dcfd333cd added region check for subnet
new cf6f85fbd fixed conflict
new df4b173d7 fixed conflict
new d3652acef [DATALAB-3073]: added keycloak client for jupyter-gpu on gcp
new f674ac727 [DATALAB-3073]: moved client creation after image creation
new df538ccbe [DATALAB-3073]: added client creation removal for azure
new 3bbcc4583 [DATALAB-3073]: removed instance terminate if client creation fails
new d4b895a04 [DATALAB-3073]: added client creation and termination for aws
new 21a39f1c1 [DATALAB-3073]: added import
new 25a448aa3 [DATALAB-3073]: added variable
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/base/scripts/configure_keycloak.py | 22 +++++++++--
.../src/general/conf/datalab.ini | 4 +-
.../src/general/lib/os/fab.py | 32 +++++++++-------
.../scripts/aws/common_terminate_notebook.py | 44 ++++++++++++++++++++++
.../src/general/scripts/aws/jupyter_configure.py | 39 +++++++++++++++++++
.../scripts/azure/common_terminate_notebook.py | 39 +++++++++++++++++++
.../src/general/scripts/azure/jupyter_configure.py | 39 +++++++++++++++++++
.../scripts/gcp/common_terminate_notebook.py | 44 ++++++++++++++++++++++
.../general/scripts/gcp/jupyter-gpu_configure.py | 40 ++++++++++++++++++++
.../src/general/scripts/gcp/jupyter_configure.py | 39 +++++++++++++++++++
10 files changed, 325 insertions(+), 17 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/09: fixed conflict
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit cf6f85fbd354a10e4f371c68f928ee30a84b461f
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Oct 11 17:59:39 2022 +0300
fixed conflict
---
.../src/base/scripts/configure_keycloak.py | 20 ++++++++---
.../src/general/lib/os/fab.py | 32 ++++++++++-------
.../src/general/scripts/gcp/jupyter_configure.py | 41 ++++++++++++++++++++++
3 files changed, 76 insertions(+), 17 deletions(-)
diff --git a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
index 5974b2c45..516a8ab52 100644
--- a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
+++ b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
@@ -40,6 +40,7 @@ parser.add_argument('--instance_public_ip', type=str, default='')
parser.add_argument('--hostname', type=str, default='')
parser.add_argument('--project_name', type=str, default='')
parser.add_argument('--endpoint_name', type=str, default='')
+parser.add_argument('--exploratory_name', type=str, default='')
args = parser.parse_args()
##############
@@ -50,6 +51,7 @@ if __name__ == "__main__":
logging.info('[CONFIGURE KEYCLOAK]')
keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format(
args.keycloak_auth_server_url)
+
keycloak_auth_data = {
"username": args.keycloak_user,
"password": args.keycloak_user_password,
@@ -63,26 +65,36 @@ if __name__ == "__main__":
keycloak_client_create_url = '{0}/admin/realms/{1}/clients'.format(args.keycloak_auth_server_url,
args.keycloak_realm_name)
if args.project_name and args.endpoint_name:
- keycloak_client_name = "{0}-{1}-{2}".format(args.service_base_name, args.project_name, args.endpoint_name)
+ if args.exploratory_name:
+ keycloak_client_name = "{0}-{1}-{2}-{3}".format(args.service_base_name, args.project_name,
+ args.endpoint_name, args.exploratory_name)
+ else:
+ keycloak_client_name = "{0}-{1}-{2}".format(args.service_base_name, args.project_name,
+ args.endpoint_name)
else:
keycloak_client_name = "{0}-ui".format(args.service_base_name)
+
keycloak_client_id = str(uuid.uuid4())
- if args.hostname == '':
+
+ if not args.hostname:
keycloak_redirectUris = 'https://{0}/*,http://{0}/*'.format(args.instance_public_ip).lower().split(',')
else:
keycloak_redirectUris = 'https://{0}/*,http://{0}/*,https://{1}/*,http://{1}/*'.format(
args.instance_public_ip, args.hostname).lower().split(',')
+
keycloak_client_data = {
"clientId": keycloak_client_name,
"id": keycloak_client_id,
"enabled": "true",
- "redirectUris": keycloak_redirectUris,
"publicClient": "false",
"secret": args.keycloak_client_secret,
"protocol": "openid-connect",
}
- if not args.project_name:
+ if not args.exploratory_name:
+ keycloak_client_data["redirectUris"] = keycloak_redirectUris
+
+ if args.exploratory_name or not args.project_name:
keycloak_client_data["serviceAccountsEnabled"] = "true"
try:
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index b4f93a218..d32fd54bc 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -40,22 +40,28 @@ from patchwork import files
# general functions for all resources
-def init_datalab_connection(hostname, username, keyfile):
+
+def init_datalab_connection(hostname, username, keyfile, reserve_user='', run_echo=True):
try:
global conn
- attempt = 0
- while attempt < 15:
- logging.info('connection attempt {}'.format(attempt))
- conn = Connection(host=hostname, user=username, connect_kwargs={'banner_timeout': 200,
+ if reserve_user:
+ users = [username, reserve_user]
+ else:
+ users = [username]
+ for user in users:
+ attempt = 0
+ while attempt < 15:
+ logging.info('connection attempt {} with user {}'.format(attempt, user))
+ conn = Connection(host=hostname, user=user, connect_kwargs={'banner_timeout': 200,
'key_filename': keyfile})
- conn.config.run.echo = True
- try:
- conn.run('hostname')
- conn.config.run.echo = True
- return conn
- except:
- attempt += 1
- time.sleep(10)
+ conn.config.run.echo = run_echo
+ try:
+ conn.run('hostname')
+ conn.config.run.echo = run_echo
+ return conn
+ except:
+ attempt += 1
+ time.sleep(10)
if attempt == 15:
logging.info('Unable to establish connection')
raise Exception
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index caa17e17d..5e972b84e 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -31,6 +31,7 @@ import sys
import traceback
import subprocess
from fabric import *
+import uuid
if __name__ == "__main__":
try:
@@ -205,6 +206,46 @@ if __name__ == "__main__":
datalab.fab.append_result("Failed to setup git credentials.", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
+
+ try:
+ logging.info('[SETUP KEYCLOAK CLIENT]')
+ notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\
+ .format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ notebook_config['keycloak_client_secret'] = str(uuid.uuid4())
+ keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
+ "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \
+ "--project_name {} --endpoint_name {} --exploratory_name {}"\
+ .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
+ os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'],
+ notebook_config['project_name'], notebook_config['endpoint_name'],
+ notebook_config['exploratory_name'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed setup keycloak client")
+ raise Exception
+
+ try:
+ conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'],
+ notebook_config['ssh_key_path'], '', False)
+
+ with open("/home/datalab-user/template.json") as py3kernel:
+ content = json.loads(py3kernel.read())
+ content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name']
+ content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret']
+ print(content['env'])
+ with open("/home/datalab-user/template.json", 'w') as py3kernel:
+ py3kernel.write(json.dumps(content))
+ except:
+ datalab.fab.append_result("Failed to write variables to .bashrc")
+ raise Exception
+
+ except Exception as err:
+ datalab.fab.append_result("Failed setup keycloak client ", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
if notebook_config['image_enabled'] == 'true':
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 05/09: [DATALAB-3073]: added client creation removal for azure
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit df538ccbe7e76d9a44734e4ab2f956d6abf1708b
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Oct 13 11:13:40 2022 +0300
[DATALAB-3073]: added client creation removal for azure
---
.../scripts/azure/common_terminate_notebook.py | 39 ++++++++++++++++++++
.../src/general/scripts/azure/jupyter_configure.py | 41 ++++++++++++++++++++++
2 files changed, 80 insertions(+)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
index 77ef93f41..d6396b192 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/common_terminate_notebook.py
@@ -25,6 +25,7 @@ import datalab.actions_lib
import datalab.fab
import datalab.meta_lib
import json
+import requests
from datalab.logger import logging
import os
import sys
@@ -54,6 +55,44 @@ def terminate_nb(resource_group_name, notebook_name):
datalab.fab.append_result("Failed to terminate instance", str(err))
sys.exit(1)
+ if os.environ['notebook_create_keycloak_client'] == 'True':
+ logging.info("Terminating notebook keycloak client")
+ try:
+ keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format(
+ os.environ['keycloak_auth_server_url'])
+ keycloak_client_url = '{0}/admin/realms/{1}/clients'.format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+
+ keycloak_auth_data = {
+ "username": os.environ['keycloak_user'],
+ "password": os.environ['keycloak_user_password'],
+ "grant_type": "password",
+ "client_id": "admin-cli",
+ }
+
+ client_params = {
+ "clientId": "{}-{}-{}-{}".format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ }
+
+ keycloak_token = requests.post(keycloak_auth_server_url, data=keycloak_auth_data).json()
+
+ keycloak_get_id_client = requests.get(keycloak_client_url, data=keycloak_auth_data, params=client_params,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"})
+ json_keycloak_client_id = json.loads(keycloak_get_id_client.text)
+ keycloak_id_client = json_keycloak_client_id[0]['id']
+
+ keycloak_client_delete_url = '{0}/admin/realms/{1}/clients/{2}'.format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'],
+ keycloak_id_client)
+
+ requests.delete(keycloak_client_delete_url,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"})
+ except Exception as err:
+ logging.error("Failed to remove project client from Keycloak", str(err))
+
if __name__ == "__main__":
# generating variables dictionary
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index 6f428037a..4ebbb60b3 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -31,6 +31,7 @@ import sys
import traceback
import subprocess
from fabric import *
+import uuid
if __name__ == "__main__":
try:
@@ -288,6 +289,46 @@ if __name__ == "__main__":
AzureActions.remove_instance(notebook_config['resource_group_name'], notebook_config['instance_name'])
sys.exit(1)
+ if os.environ['notebook_create_keycloak_client'] == 'True':
+ try:
+ logging.info('[SETUP KEYCLOAK CLIENT]')
+ notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\
+ .format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ notebook_config['keycloak_client_secret'] = str(uuid.uuid4())
+ keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
+ "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \
+ "--project_name {} --endpoint_name {} --exploratory_name {}"\
+ .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
+ os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'],
+ notebook_config['project_name'], notebook_config['endpoint_name'],
+ notebook_config['exploratory_name'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed setup keycloak client")
+ raise Exception
+
+ try:
+ conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'],
+ notebook_config['ssh_key_path'], '', False)
+ content = json.loads(conn.sudo("cat /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(notebook_config['datalab_ssh_user'])).stdout)
+ content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name']
+ content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret']
+ conn.sudo("echo '{}' > /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(json.dumps(content), notebook_config['datalab_ssh_user']))
+ conn.sudo('systemctl restart jupyter-notebook')
+ except:
+ datalab.fab.append_result("Failed to write variables to .bashrc")
+ raise Exception
+
+ except Exception as err:
+ datalab.fab.append_result("Failed setup keycloak client ", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
+
try:
logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
additional_info = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 07/09: [DATALAB-3073]: added client creation and termination for aws
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit d4b895a04582af65737ce125673731ff058a1a51
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Oct 13 11:16:38 2022 +0300
[DATALAB-3073]: added client creation and termination for aws
---
.../scripts/aws/common_terminate_notebook.py | 38 ++++++++++++++++++++++
.../src/general/scripts/aws/jupyter_configure.py | 38 ++++++++++++++++++++++
2 files changed, 76 insertions(+)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
index a7e92f1b9..46ea321a2 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
@@ -65,6 +65,44 @@ def terminate_nb(nb_tag_value, bucket_name, tag_name):
except:
sys.exit(1)
+ if os.environ['notebook_create_keycloak_client'] == 'True':
+ logging.info("Terminating notebook keycloak client")
+ try:
+ keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format(
+ os.environ['keycloak_auth_server_url'])
+ keycloak_client_url = '{0}/admin/realms/{1}/clients'.format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+
+ keycloak_auth_data = {
+ "username": os.environ['keycloak_user'],
+ "password": os.environ['keycloak_user_password'],
+ "grant_type": "password",
+ "client_id": "admin-cli",
+ }
+
+ client_params = {
+ "clientId": "{}-{}-{}-{}".format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ }
+
+ keycloak_token = requests.post(keycloak_auth_server_url, data=keycloak_auth_data).json()
+
+ keycloak_get_id_client = requests.get(keycloak_client_url, data=keycloak_auth_data, params=client_params,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"})
+ json_keycloak_client_id = json.loads(keycloak_get_id_client.text)
+ keycloak_id_client = json_keycloak_client_id[0]['id']
+
+ keycloak_client_delete_url = '{0}/admin/realms/{1}/clients/{2}'.format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'],
+ keycloak_id_client)
+
+ requests.delete(keycloak_client_delete_url,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"})
+ except Exception as err:
+ logging.error("Failed to remove project client from Keycloak", str(err))
+
if __name__ == "__main__":
# generating variables dictionary
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index cba99112f..57f0da8c2 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@ -296,6 +296,44 @@ if __name__ == "__main__":
datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name'])
sys.exit(1)
+ if os.environ['notebook_create_keycloak_client'] == 'True':
+ try:
+ logging.info('[SETUP KEYCLOAK CLIENT]')
+ notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\
+ .format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ notebook_config['keycloak_client_secret'] = str(uuid.uuid4())
+ keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
+ "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \
+ "--project_name {} --endpoint_name {} --exploratory_name {}"\
+ .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
+ os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'],
+ notebook_config['project_name'], notebook_config['endpoint_name'],
+ notebook_config['exploratory_name'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed setup keycloak client")
+ raise Exception
+
+ try:
+ conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'],
+ notebook_config['ssh_key_path'], '', False)
+ content = json.loads(conn.sudo("cat /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(notebook_config['datalab_ssh_user'])).stdout)
+ content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name']
+ content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret']
+ conn.sudo("echo '{}' > /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(json.dumps(content), notebook_config['datalab_ssh_user']))
+ conn.sudo('systemctl restart jupyter-notebook')
+ except:
+ datalab.fab.append_result("Failed to write variables to .bashrc")
+ raise Exception
+
+ except Exception as err:
+ datalab.fab.append_result("Failed setup keycloak client ", str(err))
+
try:
# generating output information
ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'],
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 08/09: [DATALAB-3073]: added import
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 21a39f1c1f6e8d5cce032060639b9b8dc0638c0f
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Oct 13 11:19:39 2022 +0300
[DATALAB-3073]: added import
---
infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
index 57f0da8c2..621dcf95b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_configure.py
@@ -32,6 +32,7 @@ import traceback
import subprocess
from fabric import *
from datalab.logger import logging
+import uuid
parser = argparse.ArgumentParser()
parser.add_argument('--uuid', type=str, default='')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/09: fixed conflict
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit df4b173d769d201f3c5631d739b1034005eb9eaf
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Oct 12 11:31:50 2022 +0300
fixed conflict
---
.../src/base/scripts/configure_keycloak.py | 6 +-
.../src/general/conf/datalab.ini | 4 +-
.../scripts/gcp/common_terminate_notebook.py | 44 +++++++++++++
.../src/general/scripts/gcp/jupyter_configure.py | 72 +++++++++++-----------
4 files changed, 88 insertions(+), 38 deletions(-)
diff --git a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
index 516a8ab52..449177c0a 100644
--- a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
+++ b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
@@ -94,7 +94,11 @@ if __name__ == "__main__":
if not args.exploratory_name:
keycloak_client_data["redirectUris"] = keycloak_redirectUris
- if args.exploratory_name or not args.project_name:
+ if not args.project_name:
+ keycloak_client_data["serviceAccountsEnabled"] = "true"
+
+ if args.exploratory_name:
+ keycloak_client_data["standardFlowEnabled"] = "false"
keycloak_client_data["serviceAccountsEnabled"] = "true"
try:
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 0cda67a6c..620fd78f1 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -372,7 +372,9 @@ nbconvert_version = 5.6.1
### nbformat_version
nbformat_version = 5.3.0
### jupyterlab version
-jupyterlab_version = 3.2.9
+jupyterlab_version = 3.4.3
+### jupyter keycloak client creation
+create_keycloak_client = False
#--- [emr] section contains all parameters that are using for emr provisioning ---#
[emr]
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
index db40b05e2..5acc11abc 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
@@ -25,6 +25,7 @@ import datalab.actions_lib
import datalab.fab
import datalab.meta_lib
import json
+import requests
from datalab.logger import logging
import os
import sys
@@ -73,6 +74,45 @@ def terminate_nb(instance_name, bucket_name, region, zone, user_name):
except Exception as err:
datalab.fab.append_result("Failed to terminate instance", str(err))
sys.exit(1)
+
+ if os.environ['notebook_create_keycloak_client'] == 'True':
+ logging.info("Terminating notebook keycloak client")
+ try:
+ keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format(
+ os.environ['keycloak_auth_server_url'])
+ keycloak_client_url = '{0}/admin/realms/{1}/clients'.format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+
+ keycloak_auth_data = {
+ "username": os.environ['keycloak_user'],
+ "password": os.environ['keycloak_user_password'],
+ "grant_type": "password",
+ "client_id": "admin-cli",
+ }
+
+ client_params = {
+ "clientId": "{}-{}-{}-{}".format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ }
+
+ keycloak_token = requests.post(keycloak_auth_server_url, data=keycloak_auth_data).json()
+
+ keycloak_get_id_client = requests.get(keycloak_client_url, data=keycloak_auth_data, params=client_params,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"})
+ json_keycloak_client_id = json.loads(keycloak_get_id_client.text)
+ keycloak_id_client = json_keycloak_client_id[0]['id']
+
+ keycloak_client_delete_url = '{0}/admin/realms/{1}/clients/{2}'.format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'],
+ keycloak_id_client)
+
+ requests.delete(keycloak_client_delete_url,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"})
+ except Exception as err:
+ logging.error("Failed to remove project client from Keycloak", str(err))
+
if __name__ == "__main__":
@@ -91,6 +131,10 @@ if __name__ == "__main__":
notebook_config['endpoint_name'])
notebook_config['gcp_region'] = os.environ['gcp_region']
notebook_config['gcp_zone'] = os.environ['gcp_zone']
+ try:
+ notebook_config['exploratory_name'] = (os.environ['exploratory_name']).replace('_', '-').lower()
+ except:
+ notebook_config['exploratory_name'] = ''
try:
logging.info('[TERMINATE NOTEBOOK]')
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index 5e972b84e..4fa87fb5c 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -206,46 +206,46 @@ if __name__ == "__main__":
datalab.fab.append_result("Failed to setup git credentials.", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
-
- try:
- logging.info('[SETUP KEYCLOAK CLIENT]')
- notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\
- .format(notebook_config['service_base_name'], notebook_config['project_name'],
- notebook_config['endpoint_name'], notebook_config['exploratory_name'])
- notebook_config['keycloak_client_secret'] = str(uuid.uuid4())
- keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
- "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \
- "--project_name {} --endpoint_name {} --exploratory_name {}"\
- .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'],
- os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
- os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'],
- notebook_config['project_name'], notebook_config['endpoint_name'],
- notebook_config['exploratory_name'])
- try:
- subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
- except:
- datalab.fab.append_result("Failed setup keycloak client")
- raise Exception
+ if os.environ['notebook_create_keycloak_client'] == 'True':
try:
- conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'],
- notebook_config['ssh_key_path'], '', False)
+ logging.info('[SETUP KEYCLOAK CLIENT]')
+ notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\
+ .format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ notebook_config['keycloak_client_secret'] = str(uuid.uuid4())
+ keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
+ "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \
+ "--project_name {} --endpoint_name {} --exploratory_name {}"\
+ .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
+ os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'],
+ notebook_config['project_name'], notebook_config['endpoint_name'],
+ notebook_config['exploratory_name'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed setup keycloak client")
+ raise Exception
- with open("/home/datalab-user/template.json") as py3kernel:
- content = json.loads(py3kernel.read())
- content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name']
- content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret']
- print(content['env'])
- with open("/home/datalab-user/template.json", 'w') as py3kernel:
- py3kernel.write(json.dumps(content))
- except:
- datalab.fab.append_result("Failed to write variables to .bashrc")
- raise Exception
+ try:
+ conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'],
+ notebook_config['ssh_key_path'], '', False)
+ content = json.loads(conn.sudo("cat /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(notebook_config['datalab_ssh_user'])).stdout)
+ content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name']
+ content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret']
+ conn.sudo("echo '{}' > /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(json.dumps(content), notebook_config['datalab_ssh_user']))
+ conn.sudo('systemctl restart jupyter-notebook')
+ except:
+ datalab.fab.append_result("Failed to write variables to .bashrc")
+ raise Exception
- except Exception as err:
- datalab.fab.append_result("Failed setup keycloak client ", str(err))
- GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
- sys.exit(1)
+ except Exception as err:
+ datalab.fab.append_result("Failed setup keycloak client ", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
if notebook_config['image_enabled'] == 'true':
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/09: [DATALAB-3073]: added keycloak client for jupyter-gpu on gcp
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit d3652acef9022ebd59b04b69a87cf959690b8c6b
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Oct 12 17:38:39 2022 +0300
[DATALAB-3073]: added keycloak client for jupyter-gpu on gcp
---
.../general/scripts/gcp/jupyter-gpu_configure.py | 40 ++++++++++++++++++++++
1 file changed, 40 insertions(+)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu_configure.py
index 2f1c6f1c0..7c13d08bb 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter-gpu_configure.py
@@ -31,6 +31,7 @@ import sys
import traceback
import subprocess
from fabric import *
+import uuid
if __name__ == "__main__":
try:
@@ -222,6 +223,45 @@ if __name__ == "__main__":
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
+ if os.environ['notebook_create_keycloak_client'] == 'True':
+ try:
+ logging.info('[SETUP KEYCLOAK CLIENT]')
+ notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\
+ .format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ notebook_config['keycloak_client_secret'] = str(uuid.uuid4())
+ keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
+ "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \
+ "--project_name {} --endpoint_name {} --exploratory_name {}"\
+ .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
+ os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'],
+ notebook_config['project_name'], notebook_config['endpoint_name'],
+ notebook_config['exploratory_name'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed setup keycloak client")
+ raise Exception
+
+ try:
+ conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'],
+ notebook_config['ssh_key_path'], '', False)
+ content = json.loads(conn.sudo("cat /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(notebook_config['datalab_ssh_user'])).stdout)
+ content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name']
+ content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret']
+ conn.sudo("echo '{}' > /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(json.dumps(content), notebook_config['datalab_ssh_user']))
+ conn.sudo('systemctl restart jupyter-notebook')
+ except:
+ datalab.fab.append_result("Failed to write variables to .bashrc")
+ raise Exception
+
+ except Exception as err:
+ datalab.fab.append_result("Failed setup keycloak client ", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
if notebook_config['image_enabled'] == 'true':
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 06/09: [DATALAB-3073]: removed instance terminate if client creation fails
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 3bbcc458301196ce3948e81985a6342b506d237c
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Oct 13 11:15:23 2022 +0300
[DATALAB-3073]: removed instance terminate if client creation fails
---
.../src/general/scripts/azure/jupyter_configure.py | 2 --
.../src/general/scripts/gcp/jupyter_configure.py | 2 --
2 files changed, 4 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
index 4ebbb60b3..3411fa862 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/jupyter_configure.py
@@ -326,8 +326,6 @@ if __name__ == "__main__":
except Exception as err:
datalab.fab.append_result("Failed setup keycloak client ", str(err))
- GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
- sys.exit(1)
try:
logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index 04cbcba92..17e3007ad 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -268,8 +268,6 @@ if __name__ == "__main__":
except Exception as err:
datalab.fab.append_result("Failed setup keycloak client ", str(err))
- GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
- sys.exit(1)
if os.environ['gpu_enabled'] == 'True':
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 04/09: [DATALAB-3073]: moved client creation after image creation
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit f674ac7278909b6040fe1fd7e432fd02c27f5346
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Oct 13 11:13:25 2022 +0300
[DATALAB-3073]: moved client creation after image creation
---
.../src/general/scripts/gcp/jupyter_configure.py | 48 +++++++++++-----------
1 file changed, 24 insertions(+), 24 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index 4fa87fb5c..04cbcba92 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -207,6 +207,30 @@ if __name__ == "__main__":
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
+ if notebook_config['image_enabled'] == 'true':
+ try:
+ logging.info('[CREATING IMAGE]')
+ primary_image_id = GCPMeta.get_image_by_name(notebook_config['expected_primary_image_name'])
+ if primary_image_id == '':
+ logging.info("Looks like it's first time we configure notebook server. Creating images.")
+ image_id_list = GCPActions.create_image_from_instance_disks(
+ notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
+ notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
+ notebook_config['gcp_wrapped_csek'])
+ if image_id_list and image_id_list[0] != '':
+ logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
+ else:
+ logging.info("Looks like another image creating operation for your template have been started a "
+ "moment ago.")
+ if image_id_list and image_id_list[1] != '':
+ logging.info("Image of secondary disk was successfully created. It's ID is {}".format(image_id_list[1]))
+ except Exception as err:
+ datalab.fab.append_result("Failed creating image.", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ GCPActions.remove_image(notebook_config['expected_primary_image_name'])
+ GCPActions.remove_image(notebook_config['expected_secondary_image_name'])
+ sys.exit(1)
+
if os.environ['notebook_create_keycloak_client'] == 'True':
try:
logging.info('[SETUP KEYCLOAK CLIENT]')
@@ -247,30 +271,6 @@ if __name__ == "__main__":
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
- if notebook_config['image_enabled'] == 'true':
- try:
- logging.info('[CREATING IMAGE]')
- primary_image_id = GCPMeta.get_image_by_name(notebook_config['expected_primary_image_name'])
- if primary_image_id == '':
- logging.info("Looks like it's first time we configure notebook server. Creating images.")
- image_id_list = GCPActions.create_image_from_instance_disks(
- notebook_config['expected_primary_image_name'], notebook_config['expected_secondary_image_name'],
- notebook_config['instance_name'], notebook_config['zone'], notebook_config['image_labels'],
- notebook_config['gcp_wrapped_csek'])
- if image_id_list and image_id_list[0] != '':
- logging.info("Image of primary disk was successfully created. It's ID is {}".format(image_id_list[0]))
- else:
- logging.info("Looks like another image creating operation for your template have been started a "
- "moment ago.")
- if image_id_list and image_id_list[1] != '':
- logging.info("Image of secondary disk was successfully created. It's ID is {}".format(image_id_list[1]))
- except Exception as err:
- datalab.fab.append_result("Failed creating image.", str(err))
- GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
- GCPActions.remove_image(notebook_config['expected_primary_image_name'])
- GCPActions.remove_image(notebook_config['expected_secondary_image_name'])
- sys.exit(1)
-
if os.environ['gpu_enabled'] == 'True':
try:
logging.info('[INSTALLING GPU DRIVERS]')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 09/09: [DATALAB-3073]: added variable
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 25a448aa32ee19ab5ece3a7e4a1a3e2d85d051ee
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Oct 13 11:21:08 2022 +0300
[DATALAB-3073]: added variable
---
.../src/general/scripts/aws/common_terminate_notebook.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
index 46ea321a2..2220a7900 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/common_terminate_notebook.py
@@ -117,6 +117,12 @@ if __name__ == "__main__":
notebook_config['project_name'],
notebook_config['endpoint_name']
).lower().replace('_', '-')
+
+ try:
+ notebook_config['exploratory_name'] = os.environ['exploratory_name'].lower()
+ except:
+ notebook_config['exploratory_name'] = ''
+
notebook_config['tag_name'] = notebook_config['service_base_name'] + '-tag'
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org