You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by lf...@apache.org on 2022/09/06 13:09:20 UTC
[incubator-datalab] 02/06: [DATALAB-1408]: added vpc and edge subnet usage
This is an automated email from the ASF dual-hosted git repository.
lfrolov pushed a commit to branch DATALAB-1408
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 49f5dbbc1b336d4c49ed93153aac05ca97338b9d
Author: leonidfrolov <fr...@gmail.com>
AuthorDate: Tue Sep 6 12:20:25 2022 +0300
[DATALAB-1408]: added vpc and edge subnet usage
---
.../scripts/azure/dataengine-service_create.py | 17 ++++++++++++++---
.../scripts/azure/dataengine-service_prepare.py | 19 +++++++++++++++++--
2 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
index fe93fc030..d31b001a1 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_create.py
@@ -46,6 +46,8 @@ parser.add_argument('--storage_account_key', type=str, help='')
parser.add_argument('--container_name', type=str, help='')
parser.add_argument('--tags', type=str, help='')
parser.add_argument('--public_key', type=str, help='')
+parser.add_argument('--vpc_id', type=str, help='')
+parser.add_argument('--subnet', type=str, help='')
args = parser.parse_args()
@@ -56,7 +58,7 @@ def build_hdinsight_cluster(resource_group_name, cluster_name, params):
def create_cluster_parameters(location, tags, cluster_version, cluster_login_username, password, master_instance_type,
worker_count, worker_instance_type, storage_account_name, storage_account_key,
- container_name, public_key):
+ container_name, public_key, vpc_id, subnet):
# Returns cluster parameters
@@ -92,6 +94,10 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use
]
}
)
+ ),
+ virtual_network_profile=VirtualNetworkProfile(
+ id=vpc_id,
+ subnet=subnet
)
),
Role(
@@ -107,6 +113,10 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use
]
}
)
+ ),
+ virtual_network_profile=VirtualNetworkProfile(
+ id=vpc_id,
+ subnet=subnet
)
)
]
@@ -129,12 +139,13 @@ def create_cluster_parameters(location, tags, cluster_version, cluster_login_use
##############
if __name__ == "__main__":
- parser.print_help()
+ #parser.print_help()
password = ''
params = create_cluster_parameters(args.location, json.loads(args.tags), args.cluster_version, 'datalab-user',
password, args.master_instance_type, args.worker_count,
args.worker_instance_type, args.storage_account_name, args.storage_account_key,
- args.container_name, args.public_key)
+ args.container_name, args.public_key, args.vpc_id, args.subnet)
+
build_hdinsight_cluster(args.resource_group_name, args.cluster_name, params)
logfile = '{}_creation.log'.format(args.cluster_name)
diff --git a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
index d69929508..5725c2c0c 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/dataengine-service_prepare.py
@@ -91,6 +91,20 @@ if __name__ == "__main__":
"endpoint_tag": hdinsight_conf['endpoint_name'],
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
hdinsight_conf['tag_name']: hdinsight_conf['storage_account_name_tag']}
+
+ hdinsight_conf['vpc_name'] = os.environ['azure_vpc_name']
+
+ hdinsight_conf['vpc_id'] = AzureMeta.get_vpc(hdinsight_conf['resource_group_name'],
+ hdinsight_conf['vpc_name']).id
+
+ hdinsight_conf['subnet_name'] = '{}-{}-{}-subnet'.format(hdinsight_conf['service_base_name'],
+ hdinsight_conf['project_name'],
+ hdinsight_conf['endpoint_name'])
+
+ hdinsight_conf['edge_network_id'] = AzureMeta.get_subnet(hdinsight_conf['resource_group_name'],
+ hdinsight_conf['vpc_name'],
+ hdinsight_conf['subnet_name']).id
+
except Exception as err:
datalab.fab.append_result("Failed to generate variables dictionary. Exception:" + str(err))
sys.exit(1)
@@ -125,13 +139,14 @@ if __name__ == "__main__":
"--master_instance_type {} --worker_instance_type {} " \
"--worker_count {} --storage_account_name {} " \
"--storage_account_key '{}' --container_name {} " \
- "--tags '{}' --public_key '{}'"\
+ "--tags '{}' --public_key '{}' --vpc_id {} --subnet {}"\
.format(hdinsight_conf['resource_group_name'], hdinsight_conf['cluster_name'],
hdinsight_conf['release_label'], hdinsight_conf['region'],
os.environ['hdinsight_master_instance_type'], os.environ['hdinsight_slave_instance_type'],
hdinsight_conf['hdinsight_worker_count'], hdinsight_conf['storage_account_name'],
hdinsight_conf['storage_account_key'], hdinsight_conf['container_name'],
- json.dumps(hdinsight_conf['cluster_tags']), ssh_admin_pubkey)
+ json.dumps(hdinsight_conf['cluster_tags']), ssh_admin_pubkey, hdinsight_conf['vpc_id'],
+ hdinsight_conf['edge_network_id'])
try:
subprocess.run("~/scripts/{}.py {}".format('dataengine-service_create', params), shell=True, check=True)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org