You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/11/02 08:47:03 UTC
[incubator-datalab] 02/09: fixed conflict
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch epm-v2.5.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit df4b173d769d201f3c5631d739b1034005eb9eaf
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Wed Oct 12 11:31:50 2022 +0300
fixed conflict
---
.../src/base/scripts/configure_keycloak.py | 6 +-
.../src/general/conf/datalab.ini | 4 +-
.../scripts/gcp/common_terminate_notebook.py | 44 +++++++++++++
.../src/general/scripts/gcp/jupyter_configure.py | 72 +++++++++++-----------
4 files changed, 88 insertions(+), 38 deletions(-)
diff --git a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
index 516a8ab52..449177c0a 100644
--- a/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
+++ b/infrastructure-provisioning/src/base/scripts/configure_keycloak.py
@@ -94,7 +94,11 @@ if __name__ == "__main__":
if not args.exploratory_name:
keycloak_client_data["redirectUris"] = keycloak_redirectUris
- if args.exploratory_name or not args.project_name:
+ if not args.project_name:
+ keycloak_client_data["serviceAccountsEnabled"] = "true"
+
+ if args.exploratory_name:
+ keycloak_client_data["standardFlowEnabled"] = "false"
keycloak_client_data["serviceAccountsEnabled"] = "true"
try:
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 0cda67a6c..620fd78f1 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -372,7 +372,9 @@ nbconvert_version = 5.6.1
### nbformat_version
nbformat_version = 5.3.0
### jupyterlab version
-jupyterlab_version = 3.2.9
+jupyterlab_version = 3.4.3
+### jupyter keycloak client creation
+create_keycloak_client = False
#--- [emr] section contains all parameters that are using for emr provisioning ---#
[emr]
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
index db40b05e2..5acc11abc 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_terminate_notebook.py
@@ -25,6 +25,7 @@ import datalab.actions_lib
import datalab.fab
import datalab.meta_lib
import json
+import requests
from datalab.logger import logging
import os
import sys
@@ -73,6 +74,45 @@ def terminate_nb(instance_name, bucket_name, region, zone, user_name):
except Exception as err:
datalab.fab.append_result("Failed to terminate instance", str(err))
sys.exit(1)
+
+ if os.environ['notebook_create_keycloak_client'] == 'True':
+ logging.info("Terminating notebook keycloak client")
+ try:
+ keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format(
+ os.environ['keycloak_auth_server_url'])
+ keycloak_client_url = '{0}/admin/realms/{1}/clients'.format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+
+ keycloak_auth_data = {
+ "username": os.environ['keycloak_user'],
+ "password": os.environ['keycloak_user_password'],
+ "grant_type": "password",
+ "client_id": "admin-cli",
+ }
+
+ client_params = {
+ "clientId": "{}-{}-{}-{}".format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ }
+
+ keycloak_token = requests.post(keycloak_auth_server_url, data=keycloak_auth_data).json()
+
+ keycloak_get_id_client = requests.get(keycloak_client_url, data=keycloak_auth_data, params=client_params,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"})
+ json_keycloak_client_id = json.loads(keycloak_get_id_client.text)
+ keycloak_id_client = json_keycloak_client_id[0]['id']
+
+ keycloak_client_delete_url = '{0}/admin/realms/{1}/clients/{2}'.format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'],
+ keycloak_id_client)
+
+ requests.delete(keycloak_client_delete_url,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"})
+ except Exception as err:
+ logging.error("Failed to remove project client from Keycloak", str(err))
+
if __name__ == "__main__":
@@ -91,6 +131,10 @@ if __name__ == "__main__":
notebook_config['endpoint_name'])
notebook_config['gcp_region'] = os.environ['gcp_region']
notebook_config['gcp_zone'] = os.environ['gcp_zone']
+ try:
+ notebook_config['exploratory_name'] = (os.environ['exploratory_name']).replace('_', '-').lower()
+ except:
+ notebook_config['exploratory_name'] = ''
try:
logging.info('[TERMINATE NOTEBOOK]')
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
index 5e972b84e..4fa87fb5c 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_configure.py
@@ -206,46 +206,46 @@ if __name__ == "__main__":
datalab.fab.append_result("Failed to setup git credentials.", str(err))
GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
sys.exit(1)
-
- try:
- logging.info('[SETUP KEYCLOAK CLIENT]')
- notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\
- .format(notebook_config['service_base_name'], notebook_config['project_name'],
- notebook_config['endpoint_name'], notebook_config['exploratory_name'])
- notebook_config['keycloak_client_secret'] = str(uuid.uuid4())
- keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
- "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \
- "--project_name {} --endpoint_name {} --exploratory_name {}"\
- .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'],
- os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
- os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'],
- notebook_config['project_name'], notebook_config['endpoint_name'],
- notebook_config['exploratory_name'])
- try:
- subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
- except:
- datalab.fab.append_result("Failed setup keycloak client")
- raise Exception
+ if os.environ['notebook_create_keycloak_client'] == 'True':
try:
- conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'],
- notebook_config['ssh_key_path'], '', False)
+ logging.info('[SETUP KEYCLOAK CLIENT]')
+ notebook_config['keycloak_client_name'] = '{}-{}-{}-{}'\
+ .format(notebook_config['service_base_name'], notebook_config['project_name'],
+ notebook_config['endpoint_name'], notebook_config['exploratory_name'])
+ notebook_config['keycloak_client_secret'] = str(uuid.uuid4())
+ keycloak_params = "--service_base_name {} --keycloak_auth_server_url {} --keycloak_realm_name {} " \
+ "--keycloak_user {} --keycloak_user_password {} --keycloak_client_secret {} " \
+ "--project_name {} --endpoint_name {} --exploratory_name {}"\
+ .format(notebook_config['service_base_name'], os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'], os.environ['keycloak_user'],
+ os.environ['keycloak_user_password'], notebook_config['keycloak_client_secret'],
+ notebook_config['project_name'], notebook_config['endpoint_name'],
+ notebook_config['exploratory_name'])
+ try:
+ subprocess.run("~/scripts/{}.py {}".format('configure_keycloak', keycloak_params), shell=True, check=True)
+ except:
+ datalab.fab.append_result("Failed setup keycloak client")
+ raise Exception
- with open("/home/datalab-user/template.json") as py3kernel:
- content = json.loads(py3kernel.read())
- content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name']
- content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret']
- print(content['env'])
- with open("/home/datalab-user/template.json", 'w') as py3kernel:
- py3kernel.write(json.dumps(content))
- except:
- datalab.fab.append_result("Failed to write variables to .bashrc")
- raise Exception
+ try:
+ conn = datalab.fab.init_datalab_connection(instance_hostname, notebook_config['datalab_ssh_user'],
+ notebook_config['ssh_key_path'], '', False)
+ content = json.loads(conn.sudo("cat /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(notebook_config['datalab_ssh_user'])).stdout)
+ content['env']['KEYCLOAK_CLIENT'] = notebook_config['keycloak_client_name']
+ content['env']['KEYCLOAK_SECRET'] = notebook_config['keycloak_client_secret']
+ conn.sudo("echo '{}' > /home/{}/.local/share/jupyter/kernels/py3spark_local/kernel.json"
+ .format(json.dumps(content), notebook_config['datalab_ssh_user']))
+ conn.sudo('systemctl restart jupyter-notebook')
+ except:
+ datalab.fab.append_result("Failed to write variables to .bashrc")
+ raise Exception
- except Exception as err:
- datalab.fab.append_result("Failed setup keycloak client ", str(err))
- GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
- sys.exit(1)
+ except Exception as err:
+ datalab.fab.append_result("Failed setup keycloak client ", str(err))
+ GCPActions.remove_instance(notebook_config['instance_name'], notebook_config['zone'])
+ sys.exit(1)
if notebook_config['image_enabled'] == 'true':
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org