You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/08 08:41:11 UTC
[incubator-datalab] branch DATALAB-1408 updated (bd5f36720 -> 2bddbd170)
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a change to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
from bd5f36720 [DATALAB-1408]: fixed minor bugs, changed zookeeper shape to A2
new 92f98b56f [DATALAB-1408]: cluster storage account is now deleted if cluster is not found
new d3ba2596b [DATALAB-1408]: changed where cluster password is generated
new 2bddbd170 [DATALAB-1408]: increased check interval during cluster creation and termination
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/dataengine-service/fabfile.py | 10 ++++++++--
.../src/general/lib/azure/actions_lib.py | 4 ++--
.../src/general/scripts/aws/dataengine-service_configure.py | 1 +
.../src/general/scripts/aws/dataengine-service_prepare.py | 1 +
.../general/scripts/azure/dataengine-service_configure.py | 12 +++++++++++-
.../src/general/scripts/azure/dataengine-service_create.py | 6 ++----
.../src/general/scripts/azure/dataengine-service_prepare.py | 10 ++++++++--
.../general/scripts/azure/dataengine-service_terminate.py | 9 +++++----
.../src/general/scripts/gcp/dataengine-service_prepare.py | 6 ++++++
9 files changed, 44 insertions(+), 15 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 02/03: [DATALAB-1408]: changed where cluster password is generated
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit d3ba2596b0a017c3ce5e2e5b9b9c7affc33da7cb
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Sep 8 11:40:18 2022 +0300
[DATALAB-1408]: changed where cluster password is generated
---
.../src/dataengine-service/fabfile.py | 10 ++++++++--
.../src/general/scripts/aws/dataengine-service_configure.py | 1 +
.../src/general/scripts/aws/dataengine-service_prepare.py | 1 +
.../general/scripts/azure/dataengine-service_configure.py | 12 +++++++++++-
.../src/general/scripts/azure/dataengine-service_create.py | 6 ++----
.../src/general/scripts/azure/dataengine-service_prepare.py | 10 ++++++++--
.../src/general/scripts/gcp/dataengine-service_prepare.py | 6 ++++++
7 files changed, 37 insertions(+), 9 deletions(-)
diff --git a/infrastructure-provisioning/src/dataengine-service/fabfile.py b/infrastructure-provisioning/src/dataengine-service/fabfile.py
index 74797b8e4..0bedc256d 100644
--- a/infrastructure-provisioning/src/dataengine-service/fabfile.py
+++ b/infrastructure-provisioning/src/dataengine-service/fabfile.py
@@ -25,6 +25,7 @@ import logging
import os
import sys
import uuid
+import secrets
import subprocess
from datalab.actions_lib import *
from datalab.fab import *
@@ -42,15 +43,20 @@ def run(ctx):
filename=local_log_filepath)
dataengine_service_config = dict()
dataengine_service_config['uuid'] = str(uuid.uuid4())[:5]
+ dataengine_service_config['access_password'] = secrets.token_urlsafe(32)
try:
- subprocess.run("~/scripts/{}.py --uuid {}".format('dataengine-service_prepare', dataengine_service_config['uuid']), shell=True, check=True)
+ subprocess.run("~/scripts/{}.py --uuid {} --access_password {}"
+ .format('dataengine-service_prepare', dataengine_service_config['uuid'],
+ dataengine_service_config['access_password']), shell=True, check=True)
except Exception as err:
traceback.print_exc()
append_result("Failed preparing Data Engine service.", str(err))
sys.exit(1)
try:
- subprocess.run("~/scripts/{}.py --uuid {}".format('dataengine-service_configure', dataengine_service_config['uuid']), shell=True, check=True)
+ subprocess.run("~/scripts/{}.py --uuid {} --access_password {}"
+ .format('dataengine-service_configure', dataengine_service_config['uuid'],
+ dataengine_service_config['access_password']), shell=True, check=True)
except Exception as err:
traceback.print_exc()
append_result("Failed configuring Data Engine service.", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
index 6ffbc3c03..03a906bf6 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_configure.py
@@ -37,6 +37,7 @@ import subprocess
parser = argparse.ArgumentParser()
parser.add_argument('--uuid', type=str, default='')
+parser.add_argument('--access_password', type=str, default='')
args = parser.parse_args()
diff --git a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
index e57a68861..8d94175b6 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/dataengine-service_prepare.py
@@ -36,6 +36,7 @@ from datalab.logger import logging
parser = argparse.ArgumentParser()
parser.add_argument('--uuid', type=str, default='')
+parser.add_argument('--access_password', type=str, default='')
args = parser.parse_args()
if __name__ == "__main__":
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_configure.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_configure.py
index ed0a9ab85..cc0dcfa11 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_configure.py
@@ -33,6 +33,12 @@ import traceback
import subprocess
from Crypto.PublicKey import RSA
from fabric import *
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--uuid', type=str, default='')
+parser.add_argument('--access_password', type=str, default='')
+args = parser.parse_args()
if __name__ == "__main__":
try:
@@ -73,7 +79,11 @@ if __name__ == "__main__":
logging.info("Slave node shape: {}".format(hdinsight_conf['hdinsight_slave_instance_type']))
logging.info("Instance count: {}".format(str(os.environ['hdinsight_count'])))
logging.info("URL access username: datalab-user")
- logging.info("URL access password: {}".format(os.environ['access_password']))
+ logging.info("URL access password: {}".format(args.access_password))
+ logging.info("Cluster URL: {}".format(hdinsight_conf['cluster_url']))
+ logging.info("Spark History URL: {}".format(hdinsight_conf['cluster_sparkhistory_url']))
+ logging.info("Jupyter URL: {}".format(hdinsight_conf['cluster_jupyter_url']))
+ logging.info("Zeppelin URL: {}".format(hdinsight_conf['cluster_zeppelin_url']))
with open("/root/result.json", 'w') as result:
res = {"hostname": hdinsight_conf['cluster_name'],
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
index a754e99e2..d03518bd5 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
@@ -24,7 +24,6 @@
import argparse
import json
import sys
-import secrets
import os
from datalab.actions_lib import *
from datalab.meta_lib import *
@@ -50,6 +49,7 @@ parser.add_argument('--tags', type=str, help='')
parser.add_argument('--public_key', type=str, help='')
parser.add_argument('--vpc_id', type=str, help='')
parser.add_argument('--subnet', type=str, help='')
+parser.add_argument('--access_password', type=str, help='')
args = parser.parse_args()
@@ -161,10 +161,8 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use
if __name__ == "__main__":
#parser.print_help()
- password = secrets.token_urlsafe(20)
- os.environ['access_password'] = password
params = create_cluster_parameters(args.location, json.loads(args.tags), args.cluster_version, 'datalab-user',
- password, args.master_instance_type, args.worker_count,
+ args.access_password, args.master_instance_type, args.worker_count,
args.worker_instance_type, args.storage_account_name, args.storage_account_key,
args.container_name, args.public_key, args.vpc_id, args.subnet)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
index af106da59..e0851272b 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
@@ -33,6 +33,12 @@ import subprocess
from Crypto.PublicKey import RSA
from datalab.logger import logging
from fabric import *
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--uuid', type=str, default='')
+parser.add_argument('--access_password', type=str, default='')
+args = parser.parse_args()
if __name__ == "__main__":
try:
@@ -139,14 +145,14 @@ if __name__ == "__main__":
"--master_instance_type {} --worker_instance_type {} " \
"--worker_count {} --storage_account_name {} " \
"--storage_account_key '{}' --container_name {} " \
- "--tags '{}' --public_key '{}' --vpc_id {} --subnet {}"\
+ "--tags '{}' --public_key '{}' --vpc_id {} --subnet {} --access_password {}"\
.format(hdinsight_conf['resource_group_name'], hdinsight_conf['cluster_name'],
hdinsight_conf['release_label'], hdinsight_conf['region'],
hdinsight_conf['hdinsight_master_instance_type'], hdinsight_conf['hdinsight_slave_instance_type'],
hdinsight_conf['hdinsight_worker_count'], hdinsight_conf['storage_account_name'],
hdinsight_conf['storage_account_key'], hdinsight_conf['container_name'],
json.dumps(hdinsight_conf['cluster_tags']), ssh_admin_pubkey, hdinsight_conf['vpc_id'],
- hdinsight_conf['edge_network_id'])
+ hdinsight_conf['edge_network_id'], args.access_password)
try:
subprocess.run("~/scripts/{}.py {}".format('dataengine-service_create', params), shell=True, check=True)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
index 5c8972dc0..399faa279 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine-service_prepare.py
@@ -33,6 +33,12 @@ import traceback
import subprocess
from Crypto.PublicKey import RSA
from fabric import *
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--uuid', type=str, default='')
+parser.add_argument('--access_password', type=str, default='')
+args = parser.parse_args()
if __name__ == "__main__":
try:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 01/03: [DATALAB-1408]: cluster storage account is now deleted if cluster is not found
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 92f98b56fc181a6f4ca031ffa6aae295fc0f8d54
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Sep 8 11:04:18 2022 +0300
[DATALAB-1408]: cluster storage account is now deleted if cluster is not found
---
.../src/general/scripts/azure/dataengine-service_terminate.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_terminate.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_terminate.py
index 1c7701644..cf0fc316c 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_terminate.py
@@ -66,13 +66,14 @@ if __name__ == "__main__":
try:
logging.info('[TERMINATE HDINSIGHT CLUSTER AND ASSOCIATED RESOURCES]')
try:
- cluster = AzureMeta.get_hdinsight_cluster(hdinsight_conf['resource_group_name'], hdinsight_conf['cluster_name'])
+ cluster = AzureMeta.get_hdinsight_cluster(hdinsight_conf['resource_group_name'],
+ hdinsight_conf['cluster_name'])
if cluster and cluster.properties.cluster_state == 'Running':
AzureActions.terminate_hdinsight_cluster(hdinsight_conf['resource_group_name'],
hdinsight_conf['cluster_name'])
- for storage_account in AzureMeta.list_storage_accounts(hdinsight_conf['resource_group_name']):
- if hdinsight_conf['storage_account_name_tag'] == storage_account.tags["Name"]:
- AzureActions.remove_storage_account(hdinsight_conf['resource_group_name'], storage_account.name)
+ for storage_account in AzureMeta.list_storage_accounts(hdinsight_conf['resource_group_name']):
+ if hdinsight_conf['storage_account_name_tag'] == storage_account.tags["Name"]:
+ AzureActions.remove_storage_account(hdinsight_conf['resource_group_name'], storage_account.name)
except Exception as err:
traceback.print_exc()
datalab.fab.append_result("Failed to terminate hdinsight cluster.", str(err))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org
[incubator-datalab] 03/03: [DATALAB-1408]: increased check interval during cluster creation and termination
Posted by lf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 2bddbd170a8a67bac89c0d9beecb75f7b283f230
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Thu Sep 8 11:40:53 2022 +0300
[DATALAB-1408]: increased check interval during cluster creation and termination
---
infrastructure-provisioning/src/general/lib/azure/actions_lib.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
index 3f45aa41d..6cb86ded6 100644
--- a/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/azure/actions_lib.py
@@ -1178,7 +1178,7 @@ class AzureActions:
result = self.hdinsight_client.clusters.begin_create(resource_group_name, cluster_name, cluster_parameters)
cluster = datalab.meta_lib.AzureMeta().get_hdinsight_cluster(resource_group_name, cluster_name)
while cluster.properties.cluster_state != 'Running':
- time.sleep(15)
+ time.sleep(30)
logging.info('The cluster is being provisioned... Please wait')
cluster = datalab.meta_lib.AzureMeta().get_hdinsight_cluster(resource_group_name, cluster_name)
return result
@@ -1197,7 +1197,7 @@ class AzureActions:
result = self.hdinsight_client.clusters.begin_delete(resource_group_name, cluster_name)
cluster_status = datalab.meta_lib.AzureMeta().get_hdinsight_cluster(resource_group_name, cluster_name)
while cluster_status:
- time.sleep(15)
+ time.sleep(30)
logging.info('The cluster is being terminated... Please wait')
cluster_status = datalab.meta_lib.AzureMeta().get_hdinsight_cluster(resource_group_name, cluster_name)
return result
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org