You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2021/03/15 09:10:52 UTC
[incubator-datalab] 01/01: [DATALAB-2091]: merged
2091-with-dataproc into 2091
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-2091
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit c6b990e726fc4c03f0e2f07686347807aee41968
Merge: e5fd210 390f337
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Mon Mar 15 11:10:29 2021 +0200
[DATALAB-2091]: merged 2091-with-dataproc into 2091
.../src/general/conf/datalab.ini | 10 ++++-
.../files/gcp/dataengine-service_Dockerfile | 3 ++
.../files/gcp/dataengine-service_description.json | 2 +-
.../src/general/lib/gcp/actions_lib.py | 5 +--
.../src/general/lib/os/debian/notebook_lib.py | 7 ++--
.../src/general/lib/os/fab.py | 45 +++++++++++++++-------
.../general/scripts/gcp/common_create_instance.py | 2 +-
.../scripts/gcp/dataengine-service_configure.py | 3 ++
.../scripts/gcp/dataengine-service_jars_parser.py | 4 +-
.../scripts/gcp/dataengine-service_prepare.py | 20 +++++++---
.../jupyter_dataengine-service_create_configs.py | 6 +--
.../src/general/scripts/gcp/project_prepare.py | 4 ++
.../scripts/os/common_configure_reverse_proxy.py | 1 +
.../templates/gcp/dataengine-service_cluster.json | 12 +-----
...on => dataengine-service_cluster_with_gpu.json} | 23 +++++++----
.../templates/gcp/dataengine-service_livy-env.sh | 5 +++
.../gcp/dataengine-service_livy.service} | 30 ++++++---------
.../src/jupyter/scripts/configure_jupyter_node.py | 4 +-
18 files changed, 117 insertions(+), 69 deletions(-)
diff --cc infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
index 628399f,10afde1..3c9b4e0
--- a/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/actions_lib.py
@@@ -1433,13 -1433,13 +1433,12 @@@ def configure_local_spark(jars_dir, tem
new_spark_defaults.append(property + ' ' + config['Properties'][property])
new_spark_defaults.append(param)
new_spark_defaults = set(new_spark_defaults)
- datalab.fab.conn.sudo("echo '{}' > /opt/spark/conf/spark-defaults.conf".format(datalab_header))
- datalab.fab.conn.sudo('bash -c "echo \'{}\' > /opt/spark/conf/spark-defaults.conf"'.format(datalab_header))
++ datalab.fab.conn.sudo('''bash -c "echo '{}' > /opt/spark/conf/spark-defaults.conf" '''.format(datalab_header))
for prop in new_spark_defaults:
-- prop = prop.rstrip()
- datalab.fab.conn.sudo('bash -c "echo \"{}\" >> /opt/spark/conf/spark-defaults.conf"'.format(prop))
+ datalab.fab.conn.sudo('''bash -c 'echo "{}" >> /opt/spark/conf/spark-defaults.conf' '''.format(prop))
datalab.fab.conn.sudo('sed -i "/^\s*$/d" /opt/spark/conf/spark-defaults.conf')
if spark_jars_paths:
- datalab.fab.conn.sudo('bash -c "echo \"{}\" >> /opt/spark/conf/spark-defaults.conf"'.format(spark_jars_paths))
+ datalab.fab.conn.sudo('''bash -c 'echo "{}" >> /opt/spark/conf/spark-defaults.conf' '''.format(spark_jars_paths))
except Exception as err:
print('Error:', str(err))
sys.exit(1)
diff --cc infrastructure-provisioning/src/general/lib/os/fab.py
index 30961b4,1e08a4f..f2255c6
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@@ -1015,9 -1034,9 +1033,10 @@@ def init_datalab_connection(hostname, u
while attempt < 10:
print('connection attempt {}'.format(attempt))
conn = Connection(host = hostname, user = username, connect_kwargs={'key_filename': keyfile})
+ conn.config.run.echo = True
try:
conn.run('ls')
+ conn.config.run.echo = True
return conn
except:
attempt += 1
diff --cc infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index a396195,fd1036d..91b5d32
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@@ -252,9 -252,14 +252,13 @@@ if __name__ == "__main__"
ingress_rule = dict()
ingress_rule['name'] = project_conf['fw_edge_ingress_internal']
ingress_rule['targetTags'] = [project_conf['network_tag']]
+ if os.environ['gcp_subnet_name']:
+ project_conf['ssn_subnet'] = os.environ['gcp_subnet_name']
+ else:
+ project_conf['ssn_subnet'] = '{}-subnet'.format(project_conf['service_base_name'])
ingress_rule['sourceRanges'] = [project_conf['private_subnet_cidr'],
- GCPMeta.get_subnet(project_conf['ssn_subnet'],
- project_conf['region'])['ipCidrRange']
- ]
+ GCPMeta.get_subnet(project_conf['subnet_name'],
+ project_conf['region'])['ipCidrRange']]
rules = [
{
'IPProtocol': 'all'
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org