You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/02/04 15:10:03 UTC
[incubator-datalab] branch DATALAB-2674 created (now 7182e0a)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git.
at 7182e0a [DATALAB-2607]: fixed dataproc creation for rstudio and zeppelin
This branch includes the following new commits:
new 7182e0a [DATALAB-2607]: fixed dataproc creation for rstudio and zeppelin
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/01: [DATALAB-2607]: fixed dataproc creation for rstudio and zeppelin
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2674
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 7182e0aa711c67a033ea5dff174ef9394c2e267b
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Fri Feb 4 17:09:43 2022 +0200
[DATALAB-2607]: fixed dataproc creation for rstudio and zeppelin
---
.../scripts/gcp/rstudio_install_dataengine-service_kernels.py | 5 ++---
.../scripts/gcp/zeppelin_install_dataengine-service_kernels.py | 5 ++---
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
index f098a95..127b492 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/rstudio_install_dataengine-service_kernels.py
@@ -43,7 +43,6 @@ parser.add_argument('--project_name', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
parser.add_argument('--edge_hostname', type=str, default='')
parser.add_argument('--proxy_port', type=str, default='')
-parser.add_argument('--pip_mirror', type=str, default='')
parser.add_argument('--application', type=str, default='')
args = parser.parse_args()
@@ -72,6 +71,6 @@ if __name__ == "__main__":
hadoop_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop')
conn.sudo('''bash -l -c 'echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf' ''')
conn.sudo('''bash -l -c 'echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc' ''')
- conn.sudo('''bash -l -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11}' '''
+ conn.sudo('''bash -l -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --application {10}' '''
.format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name, args.dataproc_version, spark_version, hadoop_version,
- args.region, args.project_name, args.os_user, args.pip_mirror, args.application))
+ args.region, args.project_name, args.os_user, args.application))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
index e17520a..22936d4 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/zeppelin_install_dataengine-service_kernels.py
@@ -41,7 +41,6 @@ parser.add_argument('--project_name', type=str, default='')
parser.add_argument('--os_user', type=str, default='')
parser.add_argument('--edge_hostname', type=str, default='')
parser.add_argument('--proxy_port', type=str, default='')
-parser.add_argument('--pip_mirror', type=str, default='')
parser.add_argument('--application', type=str, default='')
args = parser.parse_args()
@@ -76,7 +75,7 @@ if __name__ == "__main__":
hadoop_version = datalab.actions_lib.GCPActions().get_cluster_app_version(args.bucket, args.project_name, args.cluster_name, 'hadoop')
conn.sudo('''bash -l -c 'echo "[global]" > /etc/pip.conf; echo "proxy = $(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=)" >> /etc/pip.conf' ''')
conn.sudo('''bash -l -c 'echo "use_proxy=yes" > ~/.wgetrc; proxy=$(cat /etc/profile | grep proxy | head -n1 | cut -f2 -d=); echo "http_proxy=$proxy" >> ~/.wgetrc; echo "https_proxy=$proxy" >> ~/.wgetrc' ''')
- conn.sudo('''bash -l -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --pip_mirror {10} --application {11} --livy_version {12} --multiple_clusters {13} --r_enabled {14}' '''
+ conn.sudo('''bash -l -c 'unset http_proxy https_proxy; export gcp_project_id="{0}"; export conf_resource="{1}"; /usr/bin/python3 /usr/local/bin/create_configs.py --bucket {2} --cluster_name {3} --dataproc_version {4} --spark_version {5} --hadoop_version {6} --region {7} --user_name {8} --os_user {9} --application {10} --livy_version {11} --multiple_clusters {12} --r_enabled {13}' '''
.format(os.environ['gcp_project_id'], os.environ['conf_resource'], args.bucket, args.cluster_name, args.dataproc_version,
- spark_version, hadoop_version, args.region, args.project_name, args.os_user, args.pip_mirror, args.application,
+ spark_version, hadoop_version, args.region, args.project_name, args.os_user, args.application,
os.environ['notebook_livy_version'], os.environ['notebook_multiple_clusters'], r_enabled))
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org