You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2020/10/15 14:58:27 UTC
[incubator-datalab] branch DATALAB-2099 created (now edfff79)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2099
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at edfff79 [DATALAB-2099]: fixed some imports for dataproc creation/configuration
This branch includes the following new commits:
new ddccee4 [DATALAB-2099]: fixed usage of some libs during dataproc creation
new edfff79 [DATALAB-2099]: fixed some imports for dataproc creation/configuration
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/02: [DATALAB-2099]: fixed some imports for
dataproc creation/configuration
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2099
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit edfff7927ba1ea8769c956f0a8ee3486bd73bbd2
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Oct 15 17:58:06 2020 +0300
[DATALAB-2099]: fixed some imports for dataproc creation/configuration
---
.../scripts/gcp/rstudio_dataengine-service_create_configs.py | 8 ++++----
.../scripts/gcp/zeppelin_dataengine-service_create_configs.py | 10 +++++-----
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_dataengine-service_create_configs.py
index acb80c1..56e9c8e 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_dataengine-service_create_configs.py
@@ -85,9 +85,9 @@ if __name__ == "__main__":
else:
result = prepare(dataproc_dir, yarn_dir)
if result == False :
- actions_lib.GCPActions().jars(args, dataproc_dir)
- actions_lib.GCPActions().yarn(args, yarn_dir)
- actions_lib.GCPActions().install_dataproc_spark(args)
- actions_lib.GCPActions().spark_defaults(args)
+ datalab.actions_lib.GCPActions().jars(args, dataproc_dir)
+ datalab.actions_lib.GCPActions().yarn(args, yarn_dir)
+ datalab.actions_lib.GCPActions().install_dataproc_spark(args)
+ datalab.actions_lib.GCPActions().spark_defaults(args)
configuring_notebook(args.dataproc_version)
configure_rstudio()
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_dataengine-service_create_configs.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_dataengine-service_create_configs.py
index b4ad24d..8e94672 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_dataengine-service_create_configs.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_dataengine-service_create_configs.py
@@ -72,14 +72,14 @@ if __name__ == "__main__":
else:
result = prepare(dataproc_dir, yarn_dir)
if result == False :
- actions_lib.GCPActions().jars(args, dataproc_dir)
- actions_lib.GCPActions().yarn(args, yarn_dir)
- actions_lib.GCPActions().install_dataproc_spark(args)
- actions_lib.GCPActions().spark_defaults(args)
+ datalab.actions_lib.GCPActions().jars(args, dataproc_dir)
+ datalab.actions_lib.GCPActions().yarn(args, yarn_dir)
+ datalab.actions_lib.GCPActions().install_dataproc_spark(args)
+ datalab.actions_lib.GCPActions().spark_defaults(args)
configuring_notebook(args.dataproc_version)
if args.multiple_clusters == 'true':
install_remote_livy(args)
installing_python(args.region, args.bucket, args.user_name, args.cluster_name, args.application, args.pip_mirror)
- actions_lib.GCPActions().configure_zeppelin_dataproc_interpreter(args.dataproc_version, args.cluster_name, spark_dir, args.os_user,
+ datalab.actions_lib.GCPActions().configure_zeppelin_dataproc_interpreter(args.dataproc_version, args.cluster_name, spark_dir, args.os_user,
yarn_dir, args.bucket, args.user_name, args.multiple_clusters)
update_zeppelin_interpreters(args.multiple_clusters, args.r_enabled)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/02: [DATALAB-2099]: fixed usage of some libs
during dataproc creation
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2099
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit ddccee43227d7ea759902cdc630f0d47e7dbbc78
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Mon Oct 12 12:32:15 2020 +0300
[DATALAB-2099]: fixed usage of some libs during dataproc creation
---
.../src/general/scripts/gcp/dataengine-service_create.py | 6 +++---
.../scripts/gcp/jupyter_install_dataengine-service_kernels.py | 8 ++++----
.../scripts/gcp/rstudio_install_dataengine-service_kernels.py | 5 +++--
3 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
index 3987f01..42dd4ae 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
@@ -38,14 +38,14 @@ args = parser.parse_args()
def upload_jars_parser(args):
- if not actions_lib.GCPActions().put_to_bucket(args.bucket, '/root/scripts/dataengine-service_jars_parser.py', 'jars_parser.py'):
+ if not datalab.actions_lib.GCPActions().put_to_bucket(args.bucket, '/root/scripts/dataengine-service_jars_parser.py', 'jars_parser.py'):
print('Failed to upload jars_parser script')
raise Exception
def build_dataproc_cluster(args, cluster_name):
print("Will be created cluster: {}".format(json.dumps(params, sort_keys=True, indent=4, separators=(',', ': '))))
- return actions_lib.GCPActions().create_dataproc_cluster(cluster_name, args.region, params)
+ return datalab.actions_lib.GCPActions().create_dataproc_cluster(cluster_name, args.region, params)
def send_parser_job(args, cluster_name, cluster_version):
@@ -57,7 +57,7 @@ def send_parser_job(args, cluster_name, cluster_version):
job_body['job']['pysparkJob']['args'][5] = cluster_name
job_body['job']['pysparkJob']['args'][7] = cluster_version
job_body['job']['pysparkJob']['args'][9] = os.environ['conf_os_user']
- actions_lib.GCPActions().submit_dataproc_job(job_body)
+ datalab.actions_lib.GCPActions().submit_dataproc_job(job_body)
##############
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
index e9bd0b0..543668d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
@@ -79,13 +79,13 @@ if __name__ == "__main__":
env.key_filename = "{}".format(args.keyfile)
env.host_string = env.user + "@" + env.hosts
configure_notebook(args)
- spark_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ spark_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'spark')
- python_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ python_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'python')
- hadoop_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ hadoop_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'hadoop')
- r_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ r_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'r')
r_enabled = os.environ['notebook_r_enabled']
master_host = '{}-m'.format(args.cluster_name)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
index b1b3748..7431488 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
@@ -23,6 +23,7 @@
import argparse
import os
+from datalab.actions_lib import *
from datalab.meta_lib import *
from fabric.api import *
@@ -65,8 +66,8 @@ if __name__ == "__main__":
env.key_filename = "{}".format(args.keyfile)
env.host_string = env.user + "@" + env.hosts
configure_notebook(args)
- spark_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'spark')
- hadoop_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop')
+ spark_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'spark')
+ hadoop_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop')
sudo('echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf')
sudo('echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc')
sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11}'
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org