You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/01/09 10:15:58 UTC
[incubator-dlab] branch DLAB-1404 created (now ebfa22c)
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a change to branch DLAB-1404
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at ebfa22c [DLAB-1404] - [GCP] Simultaneously Data Engine Service creation fixed
This branch includes the following new commits:
new ebfa22c [DLAB-1404] - [GCP] Simultaneously Data Engine Service creation fixed
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: [DLAB-1404] - [GCP] Simultaneously Data
Engine Service creation fixed
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1404
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit ebfa22c9eedfd6eb4b68abf73c4e1dd14019b4bd
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Thu Jan 9 12:14:17 2020 +0200
[DLAB-1404] - [GCP] Simultaneously Data Engine Service creation fixed
---
infrastructure-provisioning/src/general/lib/gcp/meta_lib.py | 11 +++++++++++
.../src/general/scripts/gcp/dataengine-service_prepare.py | 9 ++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
index 5e7a96b..e961ff2 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
@@ -650,6 +650,17 @@ class GCPMeta:
traceback.print_exc(file=sys.stdout)
return ''
+ def dataproc_waiter(self, labels):
+ if os.path.exists(
+ '/response/.emr_creating_' + os.environ['exploratory_name']) or GCPMeta().get_not_configured_dataproc(
+ os.environ['notebook_instance_name']):
+ with hide('stderr', 'running', 'warnings'):
+ local("echo 'Some Dataproc cluster is still being created/terminated, waiting..'")
+ time.sleep(60)
+ self.dataproc_waiter(labels)
+ else:
+ return True
+
def get_dataproc_jobs(self):
jobs = []
try:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
index d6bb17a..91a8535 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
@@ -108,7 +108,14 @@ if __name__ == "__main__":
print("Will create exploratory environment with edge node as access point as following: ".format(json.dumps(dataproc_conf, sort_keys=True, indent=4, separators=(',', ': '))))
logging.info(json.dumps(dataproc_conf))
- local('touch /response/.dataproc_creating_{}'.format(os.environ['exploratory_name']))
+ try:
+ meta_lib.GCPMeta().dataproc_waiter(dataproc_conf['cluster_labels'])
+ local('touch /response/.dataproc_creating_{}'.format(os.environ['exploratory_name']))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Dataproc waiter fail.", str(err))
+ sys.exit(1)
+
local("echo Waiting for changes to propagate; sleep 10")
dataproc_cluster = json.loads(open('/root/templates/dataengine-service_cluster.json').read().decode('utf-8-sig'))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org