You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by my...@apache.org on 2021/03/17 11:08:55 UTC
[incubator-datalab] branch DATALAB-2091-with-dataproc updated:
[DataLab-2091] - [GCP] Dataproc deployment fixed
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DATALAB-2091-with-dataproc
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2091-with-dataproc by this push:
new a1a549a [DataLab-2091] - [GCP] Dataproc deployment fixed
a1a549a is described below
commit a1a549aa6a787d2f47e472c5aea897f1355ce380
Author: bodnarmykola <bo...@gmail.com>
AuthorDate: Wed Mar 17 12:58:36 2021 +0200
[DataLab-2091] - [GCP] Dataproc deployment fixed
---
.../src/general/conf/datalab.ini | 2 +-
.../src/general/lib/os/fab.py | 8 +-
.../jupyter_install_dataengine-service_kernels.py | 86 ++++++++++++++--------
.../general/scripts/os/common_configure_spark.py | 2 +-
4 files changed, 60 insertions(+), 38 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index 9f042cc..8e0a6d4 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -90,7 +90,7 @@ repository_user =
### Repository password
repository_pass =
### Repository url
-repository_address =
+repository_address =
#--- [aws] section contains all common parameters related to Amazon ---#
[aws]
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 1e08a4f..fe0b0df 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -498,16 +498,16 @@ def install_r_pkg(requisites):
def update_spark_jars(jars_dir='/opt/jars'):
try:
- configs = conn.sudo('find /opt/ /etc/ /usr/lib/ -name spark-defaults.conf -type f').stdout.split('\r\n')
- if exists(jars_dir):
+ configs = conn.sudo('find /opt/ /etc/ /usr/lib/ -name spark-defaults.conf -type f').stdout.split('\n')
+ if exists(conn, jars_dir):
for conf in filter(None, configs):
des_path = ''
- all_jars = conn.sudo('find {0} -name "*.jar"'.format(jars_dir)).stdout.split('\r\n')
+ all_jars = conn.sudo('find {0} -name "*.jar"'.format(jars_dir)).stdout.split('\n')
if ('-des-' in conf):
des_path = '/'.join(conf.split('/')[:3])
all_jars = find_des_jars(all_jars, des_path)
conn.sudo('''sed -i '/^# Generated\|^spark.jars/d' {0}'''.format(conf))
- conn.sudo('echo "# Generated spark.jars by DataLab from {0}\nspark.jars {1}" >> {2}'
+ conn.sudo(''' bash -l -c 'echo "# Generated spark.jars by DataLab from {0}\nspark.jars {1}" >> {2}' '''
.format(','.join(filter(None, [jars_dir, des_path])), ','.join(all_jars), conf))
# conn.sudo("sed -i 's/^[[:space:]]*//' {0}".format(conf))
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
index 857b810..0391d4c 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
@@ -25,6 +25,7 @@ import argparse
import os
from datalab.actions_lib import *
from datalab.meta_lib import *
+from datalab.fab import *
from fabric import *
parser = argparse.ArgumentParser()
@@ -45,29 +46,56 @@ parser.add_argument('--pip_mirror', type=str, default='')
parser.add_argument('--application', type=str, default='')
args = parser.parse_args()
-
def configure_notebook(args):
templates_dir = '/root/templates/'
files_dir = '/root/files/'
scripts_dir = '/root/scripts/'
- conn.put(templates_dir + 'sparkmagic_config_template.json', '/tmp/sparkmagic_config_template.json')
+ datalab.fab.conn.put(templates_dir + 'sparkmagic_config_template.json', '/tmp/sparkmagic_config_template.json')
# conn.put(templates_dir + 'pyspark_dataengine-service_template.json', '/tmp/pyspark_dataengine-service_template.json')
# conn.put(templates_dir + 'r_dataengine-service_template.json', '/tmp/r_dataengine-service_template.json')
# conn.put(templates_dir + 'toree_dataengine-service_template.json','/tmp/toree_dataengine-service_template.json')
- conn.put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application), '/tmp/create_configs.py')
+ datalab.fab.conn.put(scripts_dir + '{}_dataengine-service_create_configs.py'.format(args.application), '/tmp/create_configs.py')
# conn.put(files_dir + 'toree_kernel.tar.gz', '/tmp/toree_kernel.tar.gz')
# conn.put(templates_dir + 'toree_dataengine-service_templatev2.json', '/tmp/toree_dataengine-service_templatev2.json')
# conn.put(templates_dir + 'run_template.sh', '/tmp/run_template.sh')
- conn.sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
- conn.sudo('chmod 755 /usr/local/bin/create_configs.py')
- conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
- conn.run('mkdir -p /tmp/datalab_libs/')
- conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, env.host_string))
- conn.run('chmod a+x /tmp/datalab_libs/*')
- conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
- if exists(conn, '/usr/lib64'):
- conn.sudo('mkdir -p /usr/lib64/python3.8')
- conn.sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
+ datalab.fab.conn.sudo('\cp /tmp/create_configs.py /usr/local/bin/create_configs.py')
+ datalab.fab.conn.sudo('chmod 755 /usr/local/bin/create_configs.py')
+ datalab.fab.conn.sudo('mkdir -p /usr/lib/python3.8/datalab/')
+ datalab.fab.conn.run('mkdir -p /tmp/datalab_libs/')
+ host_string = args.os_user + "@" + args.notebook_ip
+ datalab.fab.conn.local('scp -i {} /usr/lib/python3.8/datalab/*.py {}:/tmp/datalab_libs/'.format(args.keyfile, host_string))
+ datalab.fab.conn.run('chmod a+x /tmp/datalab_libs/*')
+ datalab.fab.conn.sudo('mv /tmp/datalab_libs/* /usr/lib/python3.8/datalab/')
+ if exists(datalab.fab.conn, '/usr/lib64'):
+ datalab.fab.conn.sudo('mkdir -p /usr/lib64/python3.8')
+ datalab.fab.conn.sudo('ln -fs /usr/lib/python3.8/datalab /usr/lib64/python3.8/datalab')
+
+def install_sparkamagic_kernels(args):
+ try:
+ datalab.fab.conn.run('sudo jupyter nbextension enable --py --sys-prefix widgetsnbextension')
+ sparkmagic_dir = conn.sudo(''' bash -l -c 'pip3 show sparkmagic | grep "Location: "' ''').stdout.rstrip("\n\r").split(' ')[1]
+ datalab.fab.conn.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkkernel --prefix=/home/{}/.local/'.format(sparkmagic_dir, args.os_user), shell=True, check=True)
+ datalab.fab.conn.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/pysparkkernel --prefix=/home/{}/.local/'.format(sparkmagic_dir, args.os_user), shell=True, check=True)
+ datalab.fab.conn.run('sudo jupyter-kernelspec install {}/sparkmagic/kernels/sparkrkernel --prefix=/home/{}/.local/'.format(sparkmagic_dir, args.os_user), shell=True, check=True)
+ pyspark_kernel_name = 'PySpark (Python-{0} / Spark-{1} ) [{2}]'.format(args.python_version, args.spark_version,
+ args.cluster_name)
+ datalab.fab.conn.run('sed -i \'s|PySpark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/pysparkkernel/kernel.json'.format(
+ pyspark_kernel_name, args.os_user), shell=True, check=True)
+ spark_kernel_name = 'Spark (Scala-{0} / Spark-{1} ) [{2}]'.format(args.scala_version, args.spark_version,
+ args.cluster_name)
+ datalab.fab.conn.run('sed -i \'s|Spark|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkkernel/kernel.json'.format(
+ spark_kernel_name, args.os_user), shell=True, check=True)
+ sparkr_kernel_name = 'SparkR (R-{0} / Spark-{1} ) [{2}]'.format(args.r_version, args.spark_version,
+ args.cluster_name)
+ datalab.fab.conn.run('sed -i \'s|SparkR|{0}|g\' /home/{1}/.local/share/jupyter/kernels/sparkrkernel/kernel.json'.format(
+ sparkr_kernel_name, args.os_user), shell=True, check=True)
+ datalab.fab.conn.run('mkdir -p /home/' + args.os_user + '/.sparkmagic', shell=True, check=True)
+ datalab.fab.conn.run('cp -f /tmp/sparkmagic_config_template.json /home/' + args.os_user + '/.sparkmagic/config.json', shell=True, check=True)
+ datalab.fab.conn.run('sed -i \'s|LIVY_HOST|{0}|g\' /home/{1}/.sparkmagic/config.json'.format(
+ args.master_ip, args.os_user), shell=True, check=True)
+ datalab.fab.conn.run('sudo chown -R {0}:{0} /home/{0}/.sparkmagic/'.format(args.os_user), shell=True, check=True)
+ except:
+ sys.exit(1)
if __name__ == "__main__":
@@ -75,28 +103,22 @@ if __name__ == "__main__":
'/tmp/scala_version')
with open('/tmp/scala_version') as f:
scala_version = str(f.read()).replace(',', '')
- env.hosts = "{}".format(args.notebook_ip)
- env.user = args.os_user
- env.key_filename = "{}".format(args.keyfile)
- env.host_string = env.user + "@" + env.hosts
+ init_datalab_connection(args.notebook_ip, args.os_user, args.keyfile)
configure_notebook(args)
- spark_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ args.spark_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'spark')
- python_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ args.python_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'python')
- hadoop_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ args.hadoop_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'hadoop')
- r_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ args.r_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'r')
- r_enabled = os.environ['notebook_r_enabled']
+ #r_enabled = os.environ['notebook_r_enabled']
master_host = '{}-m'.format(args.cluster_name)
- master_ip = get_instance_private_ip_address(os.environ['gcp_zone'], master_host)
- conn.sudo('echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf')
- conn.sudo('echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc')
- conn.sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; '
- '/usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4}'
- ' --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} '
- '--application {11} --r_version {12} --r_enabled {13} --python_version {14} --master_ip {15} --scala_version {16}'
- .format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name,
- args.dataproc_version, spark_version, hadoop_version, args.region, args.project_name, args.os_user,
- args.pip_mirror, args.application, r_version, r_enabled, python_version, master_ip, scala_version))
+ args.master_ip = get_instance_private_ip_address(os.environ['gcp_zone'], master_host)
+ #datalab.fab.conn.sudo('''bash -c 'echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf' ''')
+ #datalab.fab.conn.sudo('''bash -c 'echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc' ''')
+ #datalab.fab.conn.sudo('''bash -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11} --r_version {12} --r_enabled {13} --python_version {14} --master_ip {15} --scala_version {16}' '''.format(os.environ['gcp_project_id'], os. [...]
+
+ install_sparkamagic_kernels(args)
+
diff --git a/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py b/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
index d9c9951..47f7b78 100644
--- a/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
+++ b/infrastructure-provisioning/src/general/scripts/os/common_configure_spark.py
@@ -42,7 +42,7 @@ args = parser.parse_args()
def update_spark_defaults_conf(spark_conf):
try:
timestamp = time.strftime("%a, %d %b %Y %H:%M:%S %Z", time.gmtime())
- configs = conn.sudo('find /opt/ /etc/ /usr/lib/ -name spark-defaults.conf -type f').stdout.split('\r\n')
+ configs = conn.sudo('find /opt/ /etc/ /usr/lib/ -name spark-defaults.conf -type f').stdout.split('\n')
for conf in filter(None, configs):
conn.sudo('''sed -i '/^# Updated/d' {0}'''.format(conf))
conn.sudo('''echo "# Updated by DATALAB at {0} >> {1}'''.format(timestamp, conf))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org