You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/03/30 06:54:48 UTC
[incubator-datalab] branch DATALAB-2091 updated (2f89119 -> 5977bcb)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
from 2f89119 [DATALAB-2091]: fixed fab unable to find tasks with underscore in name
add a1a549a [DataLab-2091] - [GCP] Dataproc deployment fixed
add eba5683 [DataLab-2091] - [GCP] Dataengine and Dataengine-Srvice deployment fixed
add bf451dc [DataLab-2091] - [GCP] Jupyter Toree local kernel fixed
new 5977bcb [DATALAB-2091]: merged 2091-with-dataproc
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/dataengine/scripts/configure_dataengine.py | 22 ++---
.../src/general/conf/datalab.ini | 2 +-
.../src/general/lib/gcp/actions_lib.py | 4 +-
.../src/general/lib/os/fab.py | 10 +--
.../jupyter_install_dataengine-service_kernels.py | 89 ++++++++++++--------
.../general/scripts/os/common_configure_spark.py | 2 +-
.../os/jupyter_install_dataengine_kernels.py | 96 +++++++++++++++-------
.../src/jupyter/scripts/configure_jupyter_node.py | 2 +-
8 files changed, 142 insertions(+), 85 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2091]: merged 2091-with-dataproc
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 5977bcbdc83438df7231fee2e437793a43b1ae51
Merge: 2f89119 bf451dc
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Mar 30 09:54:27 2021 +0300
[DATALAB-2091]: merged 2091-with-dataproc
.../src/dataengine/scripts/configure_dataengine.py | 22 ++---
.../src/general/conf/datalab.ini | 2 +-
.../src/general/lib/gcp/actions_lib.py | 4 +-
.../src/general/lib/os/fab.py | 10 +--
.../jupyter_install_dataengine-service_kernels.py | 89 ++++++++++++--------
.../general/scripts/os/common_configure_spark.py | 2 +-
.../os/jupyter_install_dataengine_kernels.py | 96 +++++++++++++++-------
.../src/jupyter/scripts/configure_jupyter_node.py | 2 +-
8 files changed, 142 insertions(+), 85 deletions(-)
diff --cc infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 1203bba,5b36ab2..d7d789d
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@@ -1410,12 -1410,12 +1410,12 @@@ def configure_local_spark(jars_dir, tem
if memory_type == 'driver':
spark_memory = datalab.fab.get_spark_memory()
datalab.fab.conn.sudo('sed -i "/spark.*.memory/d" /opt/spark/conf/spark-defaults.conf')
- datalab.fab.conn.sudo('bash -c "echo \"spark.{0}.memory {1}m\" >> /opt/spark/conf/spark-defaults.conf"'.format(memory_type,
- spark_memory))
+ datalab.fab.conn.sudo('''bash -c 'echo "spark.{0}.memory {1}m" >> /opt/spark/conf/spark-defaults.conf' '''
+ .format(memory_type, spark_memory))
if not exists(datalab.fab.conn,'/opt/spark/conf/spark-env.sh'):
datalab.fab.conn.sudo('mv /opt/spark/conf/spark-env.sh.template /opt/spark/conf/spark-env.sh')
- java_home = datalab.fab.conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0].replace('\n','')
- datalab.fab.conn.sudo('bash -c "echo \"export JAVA_HOME=\'{}\'\" >> /opt/spark/conf/spark-env.sh"'.format(java_home))
+ java_home = datalab.fab.conn.run("update-alternatives --query java | grep -o --color=never \'/.*/java-8.*/jre\'").stdout.splitlines()[0]
+ datalab.fab.conn.sudo('''bash -l -c 'echo "export JAVA_HOME={}" >> /opt/spark/conf/spark-env.sh' '''.format(java_home))
if 'spark_configurations' in os.environ:
datalab_header = datalab.fab.conn.sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"').stdout
spark_configurations = ast.literal_eval(os.environ['spark_configurations'])
diff --cc infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
index 4681db9,a81f065..7fc3ae9
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
@@@ -75,28 -103,22 +103,21 @@@ if __name__ == "__main__"
'/tmp/scala_version')
with open('/tmp/scala_version') as f:
scala_version = str(f.read()).replace(',', '')
- env.hosts = "{}".format(args.notebook_ip)
- env.user = args.os_user
- env.key_filename = "{}".format(args.keyfile)
- env.host_string = env.user + "@" + env.hosts
+ init_datalab_connection(args.notebook_ip, args.os_user, args.keyfile)
configure_notebook(args)
- spark_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ args.spark_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'spark')
- python_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ args.python_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'python')
- hadoop_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ args.hadoop_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'hadoop')
- r_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ args.r_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'r')
- r_enabled = os.environ['notebook_r_enabled']
+ #r_enabled = os.environ['notebook_r_enabled']
master_host = '{}-m'.format(args.cluster_name)
- master_ip = get_instance_private_ip_address(os.environ['gcp_zone'], master_host)
- conn.sudo('echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf')
- conn.sudo('''bash -l -c 'echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc' ''')
- conn.sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; '
- '/usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4}'
- ' --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} '
- '--application {11} --r_version {12} --r_enabled {13} --python_version {14} --master_ip {15} --scala_version {16}'
- .format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name,
- args.dataproc_version, spark_version, hadoop_version, args.region, args.project_name, args.os_user,
- args.pip_mirror, args.application, r_version, r_enabled, python_version, master_ip, scala_version))
+ args.master_ip = get_instance_private_ip_address(os.environ['gcp_zone'], master_host)
+ #datalab.fab.conn.sudo('''bash -c 'echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf' ''')
+ #datalab.fab.conn.sudo('''bash -c 'echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc' ''')
+ #datalab.fab.conn.sudo('''bash -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11} --r_version {12} --r_enabled {13} --python_version {14} --master_ip {15} --scala_version {16}' '''.format(os.environ['gcp_project_id'], os [...]
+
+ install_sparkamagic_kernels(args)
-
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org