You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/01/17 12:10:34 UTC
[incubator-dlab] 01/02: [DLAB-1409] - AWS Scala version is not
actual on Jupyter UI for DES/Jupyter fixed
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1409
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 46e03c12fe278182e77dfed3c1f4d219eee58f67
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Fri Jan 17 14:08:24 2020 +0200
[DLAB-1409] - AWS Scala version is not actual on Jupyter UI for DES/Jupyter fixed
---
.../scripts/post_deployment_configuration.sh | 30 ++++++++++++++++++++++
.../scripts/aws/dataengine-service_jars_parser.py | 13 ++++++++++
.../jupyter_dataengine-service_create_configs.py | 5 ++--
.../jupyter_install_dataengine-service_kernels.py | 7 ++++-
4 files changed, 52 insertions(+), 3 deletions(-)
diff --git a/infrastructure-provisioning/scripts/post_deployment_configuration.sh b/infrastructure-provisioning/scripts/post_deployment_configuration.sh
new file mode 100644
index 0000000..6bbc867
--- /dev/null
+++ b/infrastructure-provisioning/scripts/post_deployment_configuration.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+server_external_ip=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
+sed -i "s|SERVER_IP|$server_external_ip|g" /etc/nginx/conf.d/nginx_proxy.conf
+
+dlab_sbn=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/name)
+
+KEYCLOAK_REDIRECTURI='http://'$server_external_ip
+KEYCLOAK_REALM_NAME='dlab'
+KEYCLOAK_AUTH_SERVER_URL='https://idp.demo.dlabanalytics.com/auth'
+KEYCLOAK_CLIENT_NAME=$dlab_sbn'-ui'
+KEYCLOAK_CLIENT_SECRET=$(uuidgen)
+
+sed -i "s|KEYCLOAK_REDIRECTURI|$KEYCLOAK_REDIRECTURI|g" /opt/dlab/conf/self-service.yml
+sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/self-service.yml
+sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/self-service.yml
+sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/self-service.yml
+sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/self-service.yml
+
+sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/provisioning.yml
+sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/provisioning.yml
+sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/provisioning.yml
+sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/provisioning.yml
+
+ssn_subnetId=$
+
+
+
+sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/provisioning.yml
+sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/provisioning.yml
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
index f011e6a..5b46445 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
@@ -53,6 +53,11 @@ if __name__ == "__main__":
endpoint = "https://s3.{}.amazonaws.com.cn".format(args.region)
else:
endpoint = "https://s3-{}.amazonaws.com".format(args.region)
+ os.system('touch /tmp/scala_version')
+ scala_ver = subprocess.check_output("spark-submit --version 2>&1 | grep -o -P 'Scala version \K.{0,7}'",
+ shell=True).decode('UTF-8')
+ with open('/tmp/scala_version', 'w') as outfile:
+ outfile.write(scala_ver)
os.system('touch /tmp/python_version')
python_ver = subprocess.check_output("python3.5 -V 2>/dev/null | awk '{print $2}'", shell=True)
if python_ver != '':
@@ -126,4 +131,12 @@ if __name__ == "__main__":
format(args.bucket,
args.user_name,
args.cluster_name,
+ endpoint, args.region))
+ os.system('aws s3 cp /tmp/scala_version '
+ 's3://{}/{}/{}/ '
+ '--endpoint-url {} '
+ '--region {} --sse AES256'.
+ format(args.bucket,
+ args.user_name,
+ args.cluster_name,
endpoint, args.region))
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
index 7103301..b1dfa90 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
@@ -51,6 +51,7 @@ parser.add_argument('--pip_mirror', type=str, default='')
parser.add_argument('--numpy_version', type=str, default='')
parser.add_argument('--application', type=str, default='')
parser.add_argument('--r_enabled', type=str, default='')
+parser.add_argument('--scala_version', type=str, default='')
args = parser.parse_args()
emr_dir = '/opt/' + args.emr_version + '/jars/'
@@ -96,7 +97,7 @@ def toree_kernel(args):
text = text.replace('SPARK_VERSION', 'Spark-' + args.spark_version)
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('DATAENGINE-SERVICE_VERSION', args.emr_version)
- text = text.replace('SCALA_VERSION', scala_version)
+ text = text.replace('SCALA_VERSION', args.scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/kernel_var.json')
@@ -117,7 +118,7 @@ def toree_kernel(args):
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('OS_USER', args.os_user)
text = text.replace('DATAENGINE-SERVICE_VERSION', args.emr_version)
- text = text.replace('SCALA_VERSION', scala_version)
+ text = text.replace('SCALA_VERSION', args.scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/kernel_var.json')
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
index 6900a6a..146eaff 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
@@ -79,8 +79,13 @@ if __name__ == "__main__":
hadoop_version = get_hadoop_version(args.cluster_name)
r_enabled = os.environ['notebook_r_enabled']
numpy_version = os.environ['notebook_numpy_version']
+ s3_client = boto3.client('s3', config=Config(signature_version='s3v4'), region_name=args.region)
+ s3_client.download_file(args.bucket, args.project_name + '/' + args.cluster_name + '/scala_version',
+ '/tmp/scala_version')
+ with file('/tmp/scala_version') as f:
+ scala_version = str(f.read()).replace(',', '')
sudo("/usr/bin/python /usr/local/bin/jupyter_dataengine-service_create_configs.py --bucket " + args.bucket +
" --cluster_name " + args.cluster_name + " --emr_version " + args.emr_version + " --spark_version " +
spark_version + " --hadoop_version " + hadoop_version + " --region " + args.region + " --excluded_lines '"
+ args.emr_excluded_spark_properties + "' --project_name " + args.project_name + " --os_user " + args.os_user +
- " --pip_mirror " + args.pip_mirror + " --numpy_version " + numpy_version + " --application " + args.application + " --r_enabled " + r_enabled)
+ " --pip_mirror " + args.pip_mirror + " --numpy_version " + numpy_version + " --application " + args.application + " --r_enabled " + r_enabled + " --scala_version " + scala_version)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org