You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/04/15 09:49:22 UTC
[incubator-datalab] branch DATALAB-2091 updated: [DATALAB-2348]:
fixed spark reconfiguration on existing notebook
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
The following commit(s) were added to refs/heads/DATALAB-2091 by this push:
new 0bbc5ed [DATALAB-2348]: fixed spark reconfiguration on existing notebook
0bbc5ed is described below
commit 0bbc5ede919ff02111ef126e7f3d1297ffdb650d
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Apr 15 12:49:09 2021 +0300
[DATALAB-2348]: fixed spark reconfiguration on existing notebook
---
infrastructure-provisioning/src/general/lib/aws/actions_lib.py | 2 +-
infrastructure-provisioning/src/general/lib/azure/actions_lib.py | 2 +-
infrastructure-provisioning/src/general/lib/gcp/actions_lib.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 631437a..17d3449 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1684,7 +1684,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
new_spark_defaults.append(property + ' ' + config['Properties'][property])
new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- datalab.fab.conn.sudo("echo '{}' > /opt/spark/conf/spark-defaults.conf".format(datalab_header))
+ datalab.fab.conn.sudo('''bash -c 'echo "{}" > /opt/spark/conf/spark-defaults.conf' '''.format(datalab_header))
for prop in new_spark_defaults:
prop = prop.rstrip()
datalab.fab.conn.sudo('''bash -c 'echo "{}" >> /opt/spark/conf/spark-defaults.conf' '''.format(prop))
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index ac5d961..47d39cb 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1166,7 +1166,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
new_spark_defaults.append(property + ' ' + config['Properties'][property])
new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- conn.sudo("echo '{}' > /opt/spark/conf/spark-defaults.conf".format(datalab_header))
+ conn.sudo('''bash -c 'echo "{}" > /opt/spark/conf/spark-defaults.conf' '''.format(datalab_header))
for prop in new_spark_defaults:
prop = prop.rstrip()
conn.sudo('''bash -c 'echo "{}" >> /opt/spark/conf/spark-defaults.conf' '''.format(prop))
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 81b36e0..2d4bda8 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1433,7 +1433,7 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
new_spark_defaults.append(property + ' ' + config['Properties'][property])
new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- datalab.fab.conn.sudo('''bash -c "echo '{}' > /opt/spark/conf/spark-defaults.conf" '''.format(datalab_header))
+ datalab.fab.conn.sudo('''bash -c 'echo "{}" > /opt/spark/conf/spark-defaults.conf' '''.format(datalab_header))
for prop in new_spark_defaults:
datalab.fab.conn.sudo('''bash -c 'echo "{}" >> /opt/spark/conf/spark-defaults.conf' '''.format(prop))
datalab.fab.conn.sudo('sed -i "/^\s*$/d" /opt/spark/conf/spark-defaults.conf')
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org