You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by om...@apache.org on 2019/04/05 13:27:12 UTC
[incubator-dlab] branch DLAB-456 created (now 0858bba)
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a change to branch DLAB-456
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 0858bba [DLAB-456]: fixed issue with header structure
This branch includes the following new commits:
new 0858bba [DLAB-456]: fixed issue with header structure
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-456]: fixed issue with header
structure
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omartushevskyi pushed a commit to branch DLAB-456
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 0858bba082403fab7644968aafb4d63bbcc61e77
Author: Oleh Martushevskyi <Ol...@epam.com>
AuthorDate: Fri Apr 5 16:26:59 2019 +0300
[DLAB-456]: fixed issue with header structure
---
.../src/general/lib/aws/actions_lib.py | 41 ++++++++++++----------
.../src/general/lib/azure/actions_lib.py | 41 ++++++++++++----------
.../src/general/lib/gcp/actions_lib.py | 41 ++++++++++++----------
3 files changed, 69 insertions(+), 54 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index f6f37d0..642d495 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1588,21 +1588,23 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
sudo('echo "spark.{0}.memory {1}m" >> /opt/spark/conf/spark-defaults.conf'.format(memory_type,
spark_memory))
if 'spark_configurations' in os.environ:
+ dlab_header = sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
spark_configurations = ast.literal_eval(os.environ['spark_configurations'])
new_spark_defaults = list()
spark_defaults = sudo('cat /opt/spark/conf/spark-defaults.conf')
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
- for config in spark_configurations:
- if config['Classification'] == 'spark-defaults':
- for property in config['Properties']:
- if property == param.split(' ')[0]:
- param = property + ' ' + config['Properties'][property]
- else:
- new_spark_defaults.append(property + ' ' + config['Properties'][property])
- new_spark_defaults.append(param)
+ if param.split(' ')[0] != '#':
+ for config in spark_configurations:
+ if config['Classification'] == 'spark-defaults':
+ for property in config['Properties']:
+ if property == param.split(' ')[0]:
+ param = property + ' ' + config['Properties'][property]
+ else:
+ new_spark_defaults.append(property + ' ' + config['Properties'][property])
+ new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- sudo('echo "" > /opt/spark/conf/spark-defaults.conf')
+ sudo("echo '{}' > /opt/spark/conf/spark-defaults.conf".format(dlab_header))
for prop in new_spark_defaults:
prop = prop.rstrip()
sudo('echo "{}" >> /opt/spark/conf/spark-defaults.conf'.format(prop))
@@ -1755,21 +1757,24 @@ def configure_dataengine_spark(cluster_name, jars_dir, cluster_dir, datalake_ena
local('cp -f /tmp/{0}/notebook_spark-defaults_local.conf {1}spark/conf/spark-defaults.conf'.format(cluster_name,
cluster_dir))
if spark_configs:
+ dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
+ capture=True)
spark_configurations = ast.literal_eval(spark_configs)
new_spark_defaults = list()
spark_defaults = local('cat {0}spark/conf/spark-defaults.conf'.format(cluster_dir), capture=True)
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
- for config in spark_configurations:
- if config['Classification'] == 'spark-defaults':
- for property in config['Properties']:
- if property == param.split(' ')[0]:
- param = property + ' ' + config['Properties'][property]
- else:
- new_spark_defaults.append(property + ' ' + config['Properties'][property])
- new_spark_defaults.append(param)
+ if param.split(' ')[0] != '#':
+ for config in spark_configurations:
+ if config['Classification'] == 'spark-defaults':
+ for property in config['Properties']:
+ if property == param.split(' ')[0]:
+ param = property + ' ' + config['Properties'][property]
+ else:
+ new_spark_defaults.append(property + ' ' + config['Properties'][property])
+ new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- local('echo "" > {0}/spark/conf/spark-defaults.conf'.format(cluster_dir))
+ local("echo '{0}' > {1}/spark/conf/spark-defaults.conf".format(dlab_header, cluster_dir))
for prop in new_spark_defaults:
prop = prop.rstrip()
local('echo "{0}" >> {1}/spark/conf/spark-defaults.conf'.format(prop, cluster_dir))
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index a8f8774..a571a28 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1123,21 +1123,23 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
sudo('echo "spark.{0}.memory {1}m" >> /opt/spark/conf/spark-defaults.conf'.format(memory_type,
spark_memory))
if 'spark_configurations' in os.environ:
+ dlab_header = sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
spark_configurations = ast.literal_eval(os.environ['spark_configurations'])
new_spark_defaults = list()
spark_defaults = sudo('cat /opt/spark/conf/spark-defaults.conf')
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
- for config in spark_configurations:
- if config['Classification'] == 'spark-defaults':
- for property in config['Properties']:
- if property == param.split(' ')[0]:
- param = property + ' ' + config['Properties'][property]
- else:
- new_spark_defaults.append(property + ' ' + config['Properties'][property])
- new_spark_defaults.append(param)
+ if param.split(' ')[0] != '#':
+ for config in spark_configurations:
+ if config['Classification'] == 'spark-defaults':
+ for property in config['Properties']:
+ if property == param.split(' ')[0]:
+ param = property + ' ' + config['Properties'][property]
+ else:
+ new_spark_defaults.append(property + ' ' + config['Properties'][property])
+ new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- sudo('echo "" > /opt/spark/conf/spark-defaults.conf')
+ sudo("echo '{}' > /opt/spark/conf/spark-defaults.conf".format(dlab_header))
for prop in new_spark_defaults:
prop = prop.rstrip()
sudo('echo "{}" >> /opt/spark/conf/spark-defaults.conf'.format(prop))
@@ -1166,21 +1168,24 @@ def configure_dataengine_spark(cluster_name, jars_dir, cluster_dir, datalake_ena
else:
local('cp -f /opt/hadoop/etc/hadoop/core-site.xml {}hadoop/etc/hadoop/core-site.xml'.format(cluster_dir))
if spark_configs:
+ dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
+ capture=True)
spark_configurations = ast.literal_eval(spark_configs)
new_spark_defaults = list()
spark_defaults = local('cat {0}spark/conf/spark-defaults.conf'.format(cluster_dir), capture=True)
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
- for config in spark_configurations:
- if config['Classification'] == 'spark-defaults':
- for property in config['Properties']:
- if property == param.split(' ')[0]:
- param = property + ' ' + config['Properties'][property]
- else:
- new_spark_defaults.append(property + ' ' + config['Properties'][property])
- new_spark_defaults.append(param)
+ if param.split(' ')[0] != '#':
+ for config in spark_configurations:
+ if config['Classification'] == 'spark-defaults':
+ for property in config['Properties']:
+ if property == param.split(' ')[0]:
+ param = property + ' ' + config['Properties'][property]
+ else:
+ new_spark_defaults.append(property + ' ' + config['Properties'][property])
+ new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- local('echo "" > {0}/spark/conf/spark-defaults.conf'.format(cluster_dir))
+ local("echo '{0}' > {1}/spark/conf/spark-defaults.conf".format(dlab_header, cluster_dir))
for prop in new_spark_defaults:
prop = prop.rstrip()
local('echo "{0}" >> {1}/spark/conf/spark-defaults.conf'.format(prop, cluster_dir))
diff --git a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 5de8aa5..f32bae9 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@ -1284,21 +1284,23 @@ def configure_local_spark(jars_dir, templates_dir, memory_type='driver'):
sudo('echo "spark.{0}.memory {1}m" >> /opt/spark/conf/spark-defaults.conf'.format(memory_type,
spark_memory))
if 'spark_configurations' in os.environ:
+ dlab_header = sudo('cat /tmp/notebook_spark-defaults_local.conf | grep "^#"')
spark_configurations = ast.literal_eval(os.environ['spark_configurations'])
new_spark_defaults = list()
spark_defaults = sudo('cat /opt/spark/conf/spark-defaults.conf')
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
- for config in spark_configurations:
- if config['Classification'] == 'spark-defaults':
- for property in config['Properties']:
- if property == param.split(' ')[0]:
- param = property + ' ' + config['Properties'][property]
- else:
- new_spark_defaults.append(property + ' ' + config['Properties'][property])
- new_spark_defaults.append(param)
+ if param.split(' ')[0] != '#':
+ for config in spark_configurations:
+ if config['Classification'] == 'spark-defaults':
+ for property in config['Properties']:
+ if property == param.split(' ')[0]:
+ param = property + ' ' + config['Properties'][property]
+ else:
+ new_spark_defaults.append(property + ' ' + config['Properties'][property])
+ new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- sudo('echo "" > /opt/spark/conf/spark-defaults.conf')
+ sudo("echo '{}' > /opt/spark/conf/spark-defaults.conf".format(dlab_header))
for prop in new_spark_defaults:
prop = prop.rstrip()
sudo('echo "{}" >> /opt/spark/conf/spark-defaults.conf'.format(prop))
@@ -1392,21 +1394,24 @@ def configure_dataengine_spark(cluster_name, jars_dir, cluster_dir, datalake_ena
cluster_dir))
local('cp -f /opt/spark/conf/core-site.xml {}spark/conf/'.format(cluster_dir))
if spark_configs:
+ dlab_header = local('cat /tmp/{0}/notebook_spark-defaults_local.conf | grep "^#"'.format(cluster_name),
+ capture=True)
spark_configurations = ast.literal_eval(spark_configs)
new_spark_defaults = list()
spark_defaults = local('cat {0}spark/conf/spark-defaults.conf'.format(cluster_dir), capture=True)
current_spark_properties = spark_defaults.split('\n')
for param in current_spark_properties:
- for config in spark_configurations:
- if config['Classification'] == 'spark-defaults':
- for property in config['Properties']:
- if property == param.split(' ')[0]:
- param = property + ' ' + config['Properties'][property]
- else:
- new_spark_defaults.append(property + ' ' + config['Properties'][property])
- new_spark_defaults.append(param)
+ if param.split(' ')[0] != '#':
+ for config in spark_configurations:
+ if config['Classification'] == 'spark-defaults':
+ for property in config['Properties']:
+ if property == param.split(' ')[0]:
+ param = property + ' ' + config['Properties'][property]
+ else:
+ new_spark_defaults.append(property + ' ' + config['Properties'][property])
+ new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- local('echo "" > {0}/spark/conf/spark-defaults.conf'.format(cluster_dir))
+ local("echo '{0}' > {1}/spark/conf/spark-defaults.conf".format(dlab_header, cluster_dir))
for prop in new_spark_defaults:
prop = prop.rstrip()
local('echo "{0}" >> {1}/spark/conf/spark-defaults.conf'.format(prop, cluster_dir))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org