You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/01/17 12:10:33 UTC
[incubator-dlab] branch DLAB-1409 created (now e3299b3)
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a change to branch DLAB-1409
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at e3299b3 [DLAB-1409] - AWS Scala version is not actual on Jupyter UI for DES/Jupyter fixed
This branch includes the following new commits:
new 46e03c1 [DLAB-1409] - AWS Scala version is not actual on Jupyter UI for DES/Jupyter fixed
new e3299b3 [DLAB-1409] - AWS Scala version is not actual on Jupyter UI for DES/Jupyter fixed
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/02: [DLAB-1409] - AWS Scala version is not
actual on Jupyter UI for DES/Jupyter fixed
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1409
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 46e03c12fe278182e77dfed3c1f4d219eee58f67
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Fri Jan 17 14:08:24 2020 +0200
[DLAB-1409] - AWS Scala version is not actual on Jupyter UI for DES/Jupyter fixed
---
.../scripts/post_deployment_configuration.sh | 30 ++++++++++++++++++++++
.../scripts/aws/dataengine-service_jars_parser.py | 13 ++++++++++
.../jupyter_dataengine-service_create_configs.py | 5 ++--
.../jupyter_install_dataengine-service_kernels.py | 7 ++++-
4 files changed, 52 insertions(+), 3 deletions(-)
diff --git a/infrastructure-provisioning/scripts/post_deployment_configuration.sh b/infrastructure-provisioning/scripts/post_deployment_configuration.sh
new file mode 100644
index 0000000..6bbc867
--- /dev/null
+++ b/infrastructure-provisioning/scripts/post_deployment_configuration.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+server_external_ip=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
+sed -i "s|SERVER_IP|$server_external_ip|g" /etc/nginx/conf.d/nginx_proxy.conf
+
+dlab_sbn=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/name)
+
+KEYCLOAK_REDIRECTURI='http://'$server_external_ip
+KEYCLOAK_REALM_NAME='dlab'
+KEYCLOAK_AUTH_SERVER_URL='https://idp.demo.dlabanalytics.com/auth'
+KEYCLOAK_CLIENT_NAME=$dlab_sbn'-ui'
+KEYCLOAK_CLIENT_SECRET=$(uuidgen)
+
+sed -i "s|KEYCLOAK_REDIRECTURI|$KEYCLOAK_REDIRECTURI|g" /opt/dlab/conf/self-service.yml
+sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/self-service.yml
+sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/self-service.yml
+sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/self-service.yml
+sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/self-service.yml
+
+sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/provisioning.yml
+sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/provisioning.yml
+sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/provisioning.yml
+sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/provisioning.yml
+
+ssn_subnetId=$
+
+
+
+sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/provisioning.yml
+sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/provisioning.yml
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
index f011e6a..5b46445 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_jars_parser.py
@@ -53,6 +53,11 @@ if __name__ == "__main__":
endpoint = "https://s3.{}.amazonaws.com.cn".format(args.region)
else:
endpoint = "https://s3-{}.amazonaws.com".format(args.region)
+ os.system('touch /tmp/scala_version')
+ scala_ver = subprocess.check_output("spark-submit --version 2>&1 | grep -o -P 'Scala version \K.{0,7}'",
+ shell=True).decode('UTF-8')
+ with open('/tmp/scala_version', 'w') as outfile:
+ outfile.write(scala_ver)
os.system('touch /tmp/python_version')
python_ver = subprocess.check_output("python3.5 -V 2>/dev/null | awk '{print $2}'", shell=True)
if python_ver != '':
@@ -126,4 +131,12 @@ if __name__ == "__main__":
format(args.bucket,
args.user_name,
args.cluster_name,
+ endpoint, args.region))
+ os.system('aws s3 cp /tmp/scala_version '
+ 's3://{}/{}/{}/ '
+ '--endpoint-url {} '
+ '--region {} --sse AES256'.
+ format(args.bucket,
+ args.user_name,
+ args.cluster_name,
endpoint, args.region))
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
index 7103301..b1dfa90 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_dataengine-service_create_configs.py
@@ -51,6 +51,7 @@ parser.add_argument('--pip_mirror', type=str, default='')
parser.add_argument('--numpy_version', type=str, default='')
parser.add_argument('--application', type=str, default='')
parser.add_argument('--r_enabled', type=str, default='')
+parser.add_argument('--scala_version', type=str, default='')
args = parser.parse_args()
emr_dir = '/opt/' + args.emr_version + '/jars/'
@@ -96,7 +97,7 @@ def toree_kernel(args):
text = text.replace('SPARK_VERSION', 'Spark-' + args.spark_version)
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('DATAENGINE-SERVICE_VERSION', args.emr_version)
- text = text.replace('SCALA_VERSION', scala_version)
+ text = text.replace('SCALA_VERSION', args.scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/kernel_var.json')
@@ -117,7 +118,7 @@ def toree_kernel(args):
text = text.replace('SPARK_PATH', spark_path)
text = text.replace('OS_USER', args.os_user)
text = text.replace('DATAENGINE-SERVICE_VERSION', args.emr_version)
- text = text.replace('SCALA_VERSION', scala_version)
+ text = text.replace('SCALA_VERSION', args.scala_version)
with open(kernel_path, 'w') as f:
f.write(text)
local('touch /tmp/kernel_var.json')
diff --git a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
index 6900a6a..146eaff 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/jupyter_install_dataengine-service_kernels.py
@@ -79,8 +79,13 @@ if __name__ == "__main__":
hadoop_version = get_hadoop_version(args.cluster_name)
r_enabled = os.environ['notebook_r_enabled']
numpy_version = os.environ['notebook_numpy_version']
+ s3_client = boto3.client('s3', config=Config(signature_version='s3v4'), region_name=args.region)
+ s3_client.download_file(args.bucket, args.project_name + '/' + args.cluster_name + '/scala_version',
+ '/tmp/scala_version')
+ with file('/tmp/scala_version') as f:
+ scala_version = str(f.read()).replace(',', '')
sudo("/usr/bin/python /usr/local/bin/jupyter_dataengine-service_create_configs.py --bucket " + args.bucket +
" --cluster_name " + args.cluster_name + " --emr_version " + args.emr_version + " --spark_version " +
spark_version + " --hadoop_version " + hadoop_version + " --region " + args.region + " --excluded_lines '"
+ args.emr_excluded_spark_properties + "' --project_name " + args.project_name + " --os_user " + args.os_user +
- " --pip_mirror " + args.pip_mirror + " --numpy_version " + numpy_version + " --application " + args.application + " --r_enabled " + r_enabled)
+ " --pip_mirror " + args.pip_mirror + " --numpy_version " + numpy_version + " --application " + args.application + " --r_enabled " + r_enabled + " --scala_version " + scala_version)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 02/02: [DLAB-1409] - AWS Scala version is not
actual on Jupyter UI for DES/Jupyter fixed
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1409
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit e3299b3108e6b69fd0ed38348acbc8496c446f0f
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Fri Jan 17 14:10:04 2020 +0200
[DLAB-1409] - AWS Scala version is not actual on Jupyter UI for DES/Jupyter fixed
---
.../scripts/post_deployment_configuration.sh | 30 ----------------------
1 file changed, 30 deletions(-)
diff --git a/infrastructure-provisioning/scripts/post_deployment_configuration.sh b/infrastructure-provisioning/scripts/post_deployment_configuration.sh
deleted file mode 100644
index 6bbc867..0000000
--- a/infrastructure-provisioning/scripts/post_deployment_configuration.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-server_external_ip=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
-sed -i "s|SERVER_IP|$server_external_ip|g" /etc/nginx/conf.d/nginx_proxy.conf
-
-dlab_sbn=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/name)
-
-KEYCLOAK_REDIRECTURI='http://'$server_external_ip
-KEYCLOAK_REALM_NAME='dlab'
-KEYCLOAK_AUTH_SERVER_URL='https://idp.demo.dlabanalytics.com/auth'
-KEYCLOAK_CLIENT_NAME=$dlab_sbn'-ui'
-KEYCLOAK_CLIENT_SECRET=$(uuidgen)
-
-sed -i "s|KEYCLOAK_REDIRECTURI|$KEYCLOAK_REDIRECTURI|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/self-service.yml
-
-sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/provisioning.yml
-
-ssn_subnetId=$
-
-
-
-sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/provisioning.yml
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org