You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2020/10/15 14:58:28 UTC
[incubator-datalab] 01/02: [DATALAB-2099]: fixed usage of some libs
during dataproc creation
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2099
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit ddccee43227d7ea759902cdc630f0d47e7dbbc78
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Mon Oct 12 12:32:15 2020 +0300
[DATALAB-2099]: fixed usage of some libs during dataproc creation
---
.../src/general/scripts/gcp/dataengine-service_create.py | 6 +++---
.../scripts/gcp/jupyter_install_dataengine-service_kernels.py | 8 ++++----
.../scripts/gcp/rstudio_install_dataengine-service_kernels.py | 5 +++--
3 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
index 3987f01..42dd4ae 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_create.py
@@ -38,14 +38,14 @@ args = parser.parse_args()
def upload_jars_parser(args):
- if not actions_lib.GCPActions().put_to_bucket(args.bucket, '/root/scripts/dataengine-service_jars_parser.py', 'jars_parser.py'):
+ if not datalab.actions_lib.GCPActions().put_to_bucket(args.bucket, '/root/scripts/dataengine-service_jars_parser.py', 'jars_parser.py'):
print('Failed to upload jars_parser script')
raise Exception
def build_dataproc_cluster(args, cluster_name):
print("Will be created cluster: {}".format(json.dumps(params, sort_keys=True, indent=4, separators=(',', ': '))))
- return actions_lib.GCPActions().create_dataproc_cluster(cluster_name, args.region, params)
+ return datalab.actions_lib.GCPActions().create_dataproc_cluster(cluster_name, args.region, params)
def send_parser_job(args, cluster_name, cluster_version):
@@ -57,7 +57,7 @@ def send_parser_job(args, cluster_name, cluster_version):
job_body['job']['pysparkJob']['args'][5] = cluster_name
job_body['job']['pysparkJob']['args'][7] = cluster_version
job_body['job']['pysparkJob']['args'][9] = os.environ['conf_os_user']
- actions_lib.GCPActions().submit_dataproc_job(job_body)
+ datalab.actions_lib.GCPActions().submit_dataproc_job(job_body)
##############
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
index e9bd0b0..543668d 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/jupyter_install_dataengine-service_kernels.py
@@ -79,13 +79,13 @@ if __name__ == "__main__":
env.key_filename = "{}".format(args.keyfile)
env.host_string = env.user + "@" + env.hosts
configure_notebook(args)
- spark_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ spark_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'spark')
- python_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ python_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'python')
- hadoop_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ hadoop_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'hadoop')
- r_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name,
+ r_version = GCPActions().get_cluster_app_version(args.bucket, args.project_name,
args.cluster_name, 'r')
r_enabled = os.environ['notebook_r_enabled']
master_host = '{}-m'.format(args.cluster_name)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
index b1b3748..7431488 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
@@ -23,6 +23,7 @@
import argparse
import os
+from datalab.actions_lib import *
from datalab.meta_lib import *
from fabric.api import *
@@ -65,8 +66,8 @@ if __name__ == "__main__":
env.key_filename = "{}".format(args.keyfile)
env.host_string = env.user + "@" + env.hosts
configure_notebook(args)
- spark_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'spark')
- hadoop_version = actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop')
+ spark_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'spark')
+ hadoop_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop')
sudo('echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf')
sudo('echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc')
sudo('unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11}'
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org