You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2020/03/06 15:56:45 UTC
[incubator-dlab] branch odahu-integration updated:
[odahu-integration] Fixed scripts for Odahu deployment
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch odahu-integration
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/odahu-integration by this push:
new df99320 [odahu-integration] Fixed scripts for Odahu deployment
new 2f35122 Merge remote-tracking branch 'origin/odahu-integration' into odahu-integration
df99320 is described below
commit df9932050cec68b5a6d01709fd258536c6a65e38
Author: Demyan Mysakovets <de...@gmail.com>
AuthorDate: Fri Mar 6 17:55:49 2020 +0200
[odahu-integration] Fixed scripts for Odahu deployment
---
.../src/general/api/create.py | 8 +
.../src/general/api/start.py | 22 ++-
.../src/general/api/stop.py | 21 ++-
.../src/general/api/terminate.py | 8 +
.../src/general/conf/dlab.ini | 8 +-
.../src/general/scripts/gcp/odahu_deploy.py | 200 ++++++++++++++------
.../src/general/scripts/gcp/odahu_resume.py | 199 ++++++++++++++------
.../src/general/scripts/gcp/odahu_suspend.py | 201 +++++++++++++++------
.../src/general/scripts/gcp/odahu_terminate.py | 198 ++++++++++++++------
infrastructure-provisioning/src/odahu/fabfile.py | 15 +-
10 files changed, 623 insertions(+), 257 deletions(-)
diff --git a/infrastructure-provisioning/src/general/api/create.py b/infrastructure-provisioning/src/general/api/create.py
index b2437b0..b77cd96 100644
--- a/infrastructure-provisioning/src/general/api/create.py
+++ b/infrastructure-provisioning/src/general/api/create.py
@@ -54,6 +54,14 @@ if __name__ == "__main__":
with open("/response/{}.json".format(os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))
+ elif os.environ['conf_resource'] == 'odahu':
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['odahu_cluster_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['odahu_cluster_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
else:
reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
diff --git a/infrastructure-provisioning/src/general/api/start.py b/infrastructure-provisioning/src/general/api/start.py
index 469d5a6..11ca36f 100644
--- a/infrastructure-provisioning/src/general/api/start.py
+++ b/infrastructure-provisioning/src/general/api/start.py
@@ -48,13 +48,23 @@ if __name__ == "__main__":
reply['response']['result'] = json.loads(f.read())
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
- with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
- os.environ['request_id']), 'w') as response_file:
- response_file.write(json.dumps(reply))
+ if os.environ['conf_resource'] == 'odahu':
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['odahu_cluster_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['odahu_cluster_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
+ else:
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['project_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
try:
local('chmod 666 /response/*')
diff --git a/infrastructure-provisioning/src/general/api/stop.py b/infrastructure-provisioning/src/general/api/stop.py
index 8dc2a11..39d1f8e 100644
--- a/infrastructure-provisioning/src/general/api/stop.py
+++ b/infrastructure-provisioning/src/general/api/stop.py
@@ -49,13 +49,22 @@ if __name__ == "__main__":
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ if os.environ['conf_resource'] == 'odahu':
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['odahu_cluster_name'],
+ os.environ['request_id'])
- with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
- os.environ['request_id']), 'w') as response_file:
- response_file.write(json.dumps(reply))
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['odahu_cluster_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
+ else:
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['project_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
try:
local('chmod 666 /response/*')
diff --git a/infrastructure-provisioning/src/general/api/terminate.py b/infrastructure-provisioning/src/general/api/terminate.py
index 933af27..f637b51 100644
--- a/infrastructure-provisioning/src/general/api/terminate.py
+++ b/infrastructure-provisioning/src/general/api/terminate.py
@@ -54,6 +54,14 @@ if __name__ == "__main__":
with open("/response/{}.json".format(os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))
+ elif os.environ['conf_resource'] == 'odahu':
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['odahu_cluster_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['odahu_cluster_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
else:
reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 0cfc58e..70b7293 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -332,8 +332,6 @@ expl_instance_memory = 8000
###
allowed_cidr = 0.0.0.0/0
###
-bastion_tag = odahubastion
-###
dns_zone_name = ailifecycle-org
###
docker_repo = https://hub.docker.com/u/odahu
@@ -342,8 +340,6 @@ cidr = 172.31.0.0/24
###
grafana_admin = grafana_admin
###
-# grafana_pass =
-###
initial_node_count = 6
###
istio_helm_repo = https://storage.googleapis.com/istio-release/releases/1.2.2/charts
@@ -360,7 +356,7 @@ mlflow_toolchain_version = 1.0.0
###
jupyterlab_version = 1.0.0
###
-packager_version = 1.0.0
+packager_version = 1.0.0-rc10
###
# node_locations =
###
@@ -376,6 +372,8 @@ dns_project_id = or2-msq-epmd-legn-t1iylu
###
infra_vpc_peering = 0
###
+opa_policy = cGFja2FnZSBvZGFodQoKIyByb2xlcy5yZWdvCgphZG1pbiA6PSAiYWRtaW4iCmRhdGFfc2NpZW50aXN0IDo9ICJkYXRhX3NjaWVudGlzdCIKCiMgcmJhYy5yZWdvCgpyYmFjIDo9IHsKCWRhdGFfc2NpZW50aXN0OiBbCiAgICAJWyIqIiwgImFwaS92MS9tb2RlbC9kZXBsb3ltZW50LioiXSwKICAgIAlbIioiLCAiYXBpL3YxL21vZGVsL3BhY2thZ2luZy4qIl0sCiAgICAJWyIqIiwgImFwaS92MS9tb2RlbC90cmFpbmluZy4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3YxL2Nvbm5lY3Rpb24uKiJdLAogICAgCVsiUE9TVCIsICJhcGkvdjEvY29ubmVjdGlvbi4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3YxL3BhY2thZ2luZy9pbnRlZ3JhdGl [...]
+###
# tls_crt =
###
# tls_key =
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
index ab6ba85..d01d7fd 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
@@ -38,16 +38,13 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
-
-
print('Generating infrastructure names and tags')
odahu_conf = dict()
- odahu_conf['allowed_cidr'] = json.dumps(os.environ['odahu_allowed_cidr'].split(','))
- odahu_conf['bastion_tag'] = os.environ['odahu_bastion_tag']
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
odahu_conf['project_id'] = (os.environ['gcp_project_id'])
odahu_conf['region'] = (os.environ['gcp_region'])
odahu_conf['zone'] = (os.environ['gcp_zone'])
- odahu_conf['node_locations'] = json.dumps(GCPMeta().get_available_zones())
+ odahu_conf['node_locations'] = os.environ['odahu_node_locations'].split(',')
odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
@@ -72,10 +69,12 @@ if __name__ == "__main__":
odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
- odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'], os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['odahu_oauth_oidc_host']
odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
- odahu_conf['oauth_cookie_secret'] = id_generator()
+ odahu_conf['oauth_cookie_secret'] = base64.b64encode(id_generator(16))
odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
@@ -91,77 +90,160 @@ if __name__ == "__main__":
odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
odahu_conf['decrypt_token'] = id_generator()
odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
- response_file ="/response/odahu_{}_{}.json".format(odahu_conf['odahu_cluster_name'], os.environ['request_id'])
print('Preparing parameters file')
try:
local("cp /root/templates/profile.json /tmp/")
- local("sed -i \'s|<ALLOWED_IP_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['allowed_cidr']))
- local("sed -i \'s|<BASTION_TAG>|{}|g\' /tmp/profile.json".format(odahu_conf['bastion_tag']))
- local("sed -i \'s|<PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['project_id']))
- local("sed -i \'s|<CLUSTER_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_cluster_name']))
- local("sed -i \'s|<REGION>|{}|g\' /tmp/profile.json".format(odahu_conf['region']))
- local("sed -i \'s|<ZONE>|{}|g\' /tmp/profile.json".format(odahu_conf['zone']))
- local("sed -i \'s|<DNS_ZONE_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_zone_name']))
- local("sed -i \'s|<DOCKER_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['docker_repo']))
- local("sed -i \'s|<ODAHU_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_cidr']))
- local("sed -i \'s|<GRAFANA_ADMIN>|{}|g\' /tmp/profile.json".format(odahu_conf['grafana_admin']))
- local("sed -i \'s|<GRAFANA_PASS>|{}|g\' /tmp/profile.json".format(odahu_conf['grafana_pass']))
- local("sed -i \'s|<INITIAL_NODE_COUNT>|{}|g\' /tmp/profile.json".format(odahu_conf['initial_node_count']))
- local("sed -i \'s|<ISTIO_HELM_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['istio_helm_repo']))
- local("sed -i \'s|<HELM_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['helm_repo']))
- local("sed -i \'s|<K8S_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['k8s_version']))
- local("sed -i \'s|<ODAHU_INFRA_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_infra_version']))
- local("sed -i \'s|<ODAHUFLOW_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['odahuflow_version']))
- local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['mlflow_toolchain_version']))
- local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['jupyterlab_version']))
- local("sed -i \'s|<PACKAGER_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['packager_version']))
- local("sed -i \'s|<NODE_LOCATIONS>|{}|g\' /tmp/profile.json".format(odahu_conf['node_locations']))
- local("sed -i \'s|<NODE_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['node_version']))
- local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
- local("sed -i \'s|<VPC_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['vpc_name']))
- local("sed -i \'s|<SUBNET_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['private_subnet_name']))
- local("sed -i \'s|<OAUTH_CLIENT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_client_id']))
- local("sed -i \'s|<OAUTH_CLIENT_SECRET>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_client_secret']))
- local("sed -i \'s|<OAUTH_COOCKIE_SECRET>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_cookie_secret']))
- local("sed -i \'s|<PODS_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['pods_cidr']))
- local("sed -i \'s|<ROOT_DOMAIN>|{}|g\' /tmp/profile.json".format(odahu_conf['root_domain']))
- local("sed -i \'s|<SERVICE_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['service_cidr']))
- local("sed -i \'s|<TLS_CRT>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
- local("sed -i \'s|<TLS_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
- local("sed -i \'s|<SSH_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['ssh_key'].replace('\n', '')))
- local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_project_id']))
- local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\' /tmp/profile.json".format(odahu_conf['decrypt_token']))
- local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\' /tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
- local("sed -i \'s|\r||g\' /tmp/profile.json")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {"allowed_ips": odahu_conf['allowed_cidr'],
+ "cloud_type": "gcp",
+ "cluster_name": "{}".format(odahu_conf['odahu_cluster_name']),
+ "cluster_type": "gcp/gke",
+ "cluster_context": "",
+ "dns_zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ "docker_password": "",
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_user": "",
+ "gcp_cidr": "{}".format(odahu_conf['odahu_cidr']),
+ "gke_node_tag": "{}-gke-node".format(odahu_conf['odahu_cluster_name']).split(','),
+ "grafana_admin": "{}".format(odahu_conf['grafana_admin']),
+ "grafana_pass": "BMuYxTw6v5",
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "data_bucket": "{}-data-bucket".format(odahu_conf['odahu_cluster_name']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "odahu_infra_version": "{}".format(odahu_conf['odahu_infra_version']),
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "node_locations": odahu_conf['node_locations'],
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "OWl6VnV3SHNIQzVyMThqVw==",
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_jwks_url": "https://idp.demo.dlabanalytics.com/auth/realms/dlab/protocol/openid-connect/certs",
+ "oauth_oidc_port": 443,
+ "pods_cidr": "{}".format(odahu_conf['pods_cidr']),
+ "project_id": "{}".format(odahu_conf['project_id']),
+ "region": "{}".format(odahu_conf['region']),
+ "root_domain": "{}".format(odahu_conf['root_domain']),
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['odahu_cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "zone": "{}".format(odahu_conf['zone']),
+ "dns_project_id": "{}".format(odahu_conf['dns_project_id']),
+ "odahuflow_connection_decrypt_token": "qfY2k5bA2M",
+ "authorization_enabled": "true",
+ "authz_dry_run": "true",
+ "github_org_name": "",
+ "oauth_local_jwks": "",
+ "oauth_mesh_enabled": "false",
+ "k8s_version": "1.14.10-gke.17",
+ "opa_policies": {
+ "policy.rego": "cGFja2FnZSBvZGFodQoKIyByb2xlcy5yZWdvCgphZG1pbiA6PSAiYWRtaW4iCmRhdGFfc2NpZW50aXN0IDo9ICJkYXRhX3NjaWVudGlzdCIKCiMgcmJhYy5yZWdvCgpyYmFjIDo9IHsKCWRhdGFfc2NpZW50aXN0OiBbCiAgICAJWyIqIiwgImFwaS92MS9tb2RlbC9kZXBsb3ltZW50LioiXSwKICAgIAlbIioiLCAiYXBpL3YxL21vZGVsL3BhY2thZ2luZy4qIl0sCiAgICAJWyIqIiwgImFwaS92MS9tb2RlbC90cmFpbmluZy4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3YxL2Nvbm5lY3Rpb24uKiJdLAogICAgCVsiUE9TVCIsICJhcGkvdjEvY29ubmVjdGlvbi4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3Yx [...]
+ "node_pools": {
+ "packaging": {
+ "disk_size_gb": 64,
+ "max_node_count": 3,
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "taints": [
+ {
+ "value": "packaging",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-standard-4",
+ "disk_type": "pd-ssd"
+ },
+ "training": {
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "disk_size_gb": 100,
+ "taints": [
+ {
+ "value": "training",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-highcpu-8"
+ },
+ "main": {
+ "init_node_count": 3,
+ "disk_size_gb": 64,
+ "min_node_count": 1,
+ "max_node_count": 5
+ },
+ "training_gpu": {
+ "labels": {
+ "mode": "odahu-flow-training-gpu"
+ },
+ "gpu": [
+ {
+ "count": 2,
+ "type": "nvidia-tesla-p100"
+ }
+ ],
+ "disk_size_gb": 100,
+ "taints": [
+ {
+ "value": "training-gpu",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-standard-8"
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "taints": [
+ {
+ "value": "deployment",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "max_node_count": 3
+ }
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
local('cp /tmp/profile.json /')
- local('cat /profile.json')
except Exception as err:
traceback.print_exc()
append_result("Failed to configure parameter file.", str(err))
sys.exit(1)
try:
- local('tf_runner create -o /response/odahu_{}_{}.json'.format(odahu_conf['odahu_cluster_name'], os.environ['request_id']))
+ local('tf_runner create -o /tmp/result.json')
+ local("sed -i 's|name|description|g' /tmp/result.json")
except Exception as err:
traceback.print_exc()
append_result("Failed to deploy Odahu cluster.", str(err))
sys.exit(1)
# generating output information
- try:
- local("sed -e 's|name = |"description": |g' result")
- local("sed -e 's|url = |"url": |g' result")
- odahu_urls = local("cat result")
- except Exception as err:
- traceback.print_exc()
- append_result("Failed to generate output information.", str(err))
- sys.exit(1)
-
print('[SUMMARY]')
logging.info('[SUMMARY]')
print('Cluster name: {}'.format(odahu_conf['odahu_cluster_name']))
-
+ with open('/tmp/result.json', 'r') as f:
+ output = json.load(f)
+ odahu_urls = json.dumps(output['odahu_urls']['value'], sort_keys=True, indent=4)
+ print('Odahu urls: {}'.format(odahu_urls))
+ res = dict()
+ res['odahu_urls'] = output['odahu_urls']['value']
with open("/root/result.json", 'w') as result:
- res = {"odahu_urls": [odahu_urls]}
result.write(json.dumps(res))
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py
index 459edb8..8710495 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py
@@ -30,6 +30,7 @@ from dlab.actions_lib import *
import os
import string
import random
+import base64
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@ -39,16 +40,13 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
- secret = ''.join(random.choices(string.ascii_uppercase + +string.ascii_lowercase +
- string.digits, k = 16))
-
print('Generating infrastructure names and tags')
odahu_conf = dict()
- odahu_conf['allowed_cidr'] = json.dumps(os.environ['odahu_allowed_cidr'].split(','))
- odahu_conf['bastion_tag'] = os.environ['odahu_bastion_tag']
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
odahu_conf['project_id'] = (os.environ['gcp_project_id'])
odahu_conf['region'] = (os.environ['gcp_region'])
odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['node_locations'] = os.environ['odahu_node_locations'].split(',')
odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
@@ -68,77 +66,166 @@ if __name__ == "__main__":
odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
odahu_conf['project_name'])
odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
- odahu_conf['grafana_pass'] = os.environ['odahu_grafana_pass']
+ odahu_conf['grafana_pass'] = os.environ['grafana_pass']
odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
- odahu_conf['oauth_oidc_issuer_url'] = os.environ['oauth_oidc_issuer_url']
- odahu_conf['oauth_client_id'] = os.environ['oauth_client_id']
- odahu_conf['oauth_client_secret'] = os.environ['oauth_client_secret']
- odahu_conf['oauth_cookie_secret'] = secret
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['odahu_oauth_oidc_host']
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = os.environ['oauth_cookie_secret']
odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
odahu_conf['packager_version'] = os.environ['odahu_packager_version']
- odahu_conf['model_docker_url'] = os.environ['odahu_model_docker_url']
- odahu_conf['node_locations'] = json.dumps(os.environ['odahu_node_locations'].split(','))
odahu_conf['node_version'] = os.environ['odahu_node_version']
odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
odahu_conf['root_domain'] = os.environ['odahu_root_domain']
odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
- odahu_conf['tls_crt'] = base64.b64decode(os.environ['tls_crt'])
- odahu_conf['tls_key'] = base64.b64decode(os.environ['tls_key'])
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
odahu_conf['ssh_key'] = os.environ['ssh_key']
- odahu_conf['dns_project_id'] = os.environ['dns_project_id']
- odahu_conf['decrypt_token'] = secret
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = os.environ['decrypt_token']
odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
print('Preparing parameters file')
try:
local("cp /root/templates/profile.json /tmp/")
- local("sed -i \'s|<ALLOWED_IP_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['allowed_cidr']))
- local("sed -i \'s|<BASTION_TAG>|{}|g\' /tmp/profile.json".format(odahu_conf['bastion_tag']))
- local("sed -i \'s|<PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['project_id']))
- local("sed -i \'s|<CLUSTER_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_cluster_name']))
- local("sed -i \'s|<REGION>|{}|g\' /tmp/profile.json".format(odahu_conf['region']))
- local("sed -i \'s|<ZONE>|{}|g\' /tmp/profile.json".format(odahu_conf['zone']))
- local("sed -i \'s|<DNS_ZONE_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_zone_name']))
- local("sed -i \'s|<DOCKER_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['docker_repo']))
- local("sed -i \'s|<ODAHU_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_cidr']))
- local("sed -i \'s|<GRAFANA_ADMIN>|{}|g\' /tmp/profile.json".format(odahu_conf['grafana_admin']))
- local("sed -i \'s|<GRAFANA_PASS>|{}|g\' /tmp/profile.json".format(odahu_conf['grafana_pass']))
- local("sed -i \'s|<INITIAL_NODE_COUNT>|{}|g\' /tmp/profile.json".format(odahu_conf['initial_node_count']))
- local("sed -i \'s|<ISTIO_HELM_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['istio_helm_repo']))
- local("sed -i \'s|<HELM_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['helm_repo']))
- local("sed -i \'s|<K8S_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['k8s_version']))
- local("sed -i \'s|<ODAHU_INFRA_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_infra_version']))
- local("sed -i \'s|<ODAHUFLOW_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['odahuflow_version']))
- local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\' /tmp/profile.json".format(
- odahu_conf['mlflow_toolchain_version']))
- local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['jupyterlab_version']))
- local("sed -i \'s|<PACKAGER_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['packager_version']))
- local("sed -i \'s|<MODEL_DOCKER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['model_docker_url']))
- local("sed -i \'s|<NODE_LOCATIONS>|{}|g\' /tmp/profile.json".format(odahu_conf['node_locations']))
- local("sed -i \'s|<NODE_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['node_version']))
- local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
- local("sed -i \'s|<VPC_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['vpc_name']))
- local("sed -i \'s|<SUBNET_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['private_subnet_name']))
- local("sed -i \'s|<OAUTH_CLIENT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_client_id']))
- local("sed -i \'s|<OAUTH_CLIENT_SECRET>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_client_secret']))
- local("sed -i \'s|<OAUTH_COOCKIE_SECRET>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_cookie_secret']))
- local("sed -i \'s|<PODS_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['pods_cidr']))
- local("sed -i \'s|<ROOT_DOMAIN>|{}|g\' /tmp/profile.json".format(odahu_conf['root_domain']))
- local("sed -i \'s|<SERVICE_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['service_cidr']))
- local("sed -i \'s|<TLS_CRT>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
- local("sed -i \'s|<TLS_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
- local("sed -i \'s|<SSH_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
- local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_project_id']))
- local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\' /tmp/profile.json".format(odahu_conf['decrypt_token']))
- local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\' /tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {"allowed_ips": odahu_conf['allowed_cidr'],
+ "cloud_type": "gcp",
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "gcp/gke",
+ "dns_zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ "docker_password": "",
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_user": "",
+ "gcp_cidr": "{}".format(odahu_conf['odahu_cidr']),
+ "gke_node_tag": "{}-gke-node".format(odahu_conf['cluster_name']),
+ "grafana_admin": "{}".format(odahu_conf['grafana_admin']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "odahu_infra_version": "{}".format(odahu_conf['odahu_infra_version']),
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "node_locations": odahu_conf['node_locations'],
+ "node_labels": {
+ "name": "{}".format(odahu_conf['cluster_name']),
+ "product": "dlab",
+ "user": "{}".format(odahu_conf['edge_user_name']),
+ "sbn": "{}".format(odahu_conf['service_base_name'])
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_jwks_url": "https://idp.demo.dlabanalytics.com/auth/realms/dlab/protocol/openid-connect/certs",
+ "oauth_oidc_port": 443,
+ "oauth_local_jwks": "",
+ "oauth_mesh_enabled": "false",
+ "pods_cidr": "{}".format(odahu_conf['pods_cidr']),
+ "project_id": "{}".format(odahu_conf['project_id']),
+ "region": "{}".format(odahu_conf['region']),
+ "root_domain": "{}".format(odahu_conf['root_domain']),
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "zone": "{}".format(odahu_conf['zone']),
+ "dns_project_id": "{}".format(odahu_conf['dns_project_id']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "authorization_enabled": "true",
+ "authz_dry_run": "true",
+ "k8s_version": "1.13.12-gke.25",
+ "node_pools": {
+ "packaging": {
+ "disk_size_gb": 64,
+ "max_node_count": 3,
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "taints": [
+ {
+ "value": "packaging",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-standard-4",
+ "disk_type": "pd-ssd"
+ },
+ "training": {
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "disk_size_gb": 100,
+ "taints": [
+ {
+ "value": "training",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-highcpu-8"
+ },
+ "main": {
+ "init_node_count": 3,
+ "disk_size_gb": 64,
+ "min_node_count": 1,
+ "max_node_count": 5
+ },
+ "training_gpu": {
+ "labels": {
+ "mode": "odahu-flow-training-gpu"
+ },
+ "gpu": [
+ {
+ "count": 2,
+ "type": "nvidia-tesla-p100"
+ }
+ ],
+ "disk_size_gb": 100,
+ "taints": [
+ {
+ "value": "training-gpu",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-standard-8"
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "taints": [
+ {
+ "value": "deployment",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "max_node_count": 3
+ }
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
local('cp /tmp/profile.json /')
- local('cat /profile.json')
except Exception as err:
traceback.print_exc()
append_result("Failed to configure parameter file.", str(err))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py
index e967315..b2f3894 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py
@@ -28,6 +28,7 @@ from dlab.fab import *
from dlab.meta_lib import *
from dlab.actions_lib import *
import os
+import base64
if __name__ == "__main__":
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
@@ -37,16 +38,13 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
- secret = ''.join(random.choices(string.ascii_uppercase + +string.ascii_lowercase +
- string.digits, k=16))
-
print('Generating infrastructure names and tags')
odahu_conf = dict()
- odahu_conf['allowed_cidr'] = json.dumps(os.environ['odahu_allowed_cidr'].split(','))
- odahu_conf['bastion_tag'] = os.environ['odahu_bastion_tag']
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
odahu_conf['project_id'] = (os.environ['gcp_project_id'])
odahu_conf['region'] = (os.environ['gcp_region'])
odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['node_locations'] = os.environ['odahu_node_locations'].split(',')
odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
@@ -66,77 +64,166 @@ if __name__ == "__main__":
odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
odahu_conf['project_name'])
odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
- odahu_conf['grafana_pass'] = os.environ['odahu_grafana_pass']
+ odahu_conf['grafana_pass'] = os.environ['grafana_pass']
odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
- odahu_conf['oauth_oidc_issuer_url'] = os.environ['oauth_oidc_issuer_url']
- odahu_conf['oauth_client_id'] = os.environ['oauth_client_id']
- odahu_conf['oauth_client_secret'] = os.environ['oauth_client_secret']
- odahu_conf['oauth_cookie_secret'] = secret
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['odahu_oauth_oidc_host']
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = os.environ['oauth_cookie_secret']
odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
odahu_conf['packager_version'] = os.environ['odahu_packager_version']
- odahu_conf['model_docker_url'] = os.environ['odahu_model_docker_url']
- odahu_conf['node_locations'] = json.dumps(os.environ['odahu_node_locations'].split(','))
odahu_conf['node_version'] = os.environ['odahu_node_version']
odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
odahu_conf['root_domain'] = os.environ['odahu_root_domain']
odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
- odahu_conf['tls_crt'] = base64.b64decode(os.environ['tls_crt'])
- odahu_conf['tls_key'] = base64.b64decode(os.environ['tls_key'])
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
odahu_conf['ssh_key'] = os.environ['ssh_key']
- odahu_conf['dns_project_id'] = os.environ['dns_project_id']
- odahu_conf['decrypt_token'] = secret
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = os.environ['decrypt_token']
odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
print('Preparing parameters file')
try:
local("cp /root/templates/profile.json /tmp/")
- local("sed -i \'s|<ALLOWED_IP_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['allowed_cidr']))
- local("sed -i \'s|<BASTION_TAG>|{}|g\' /tmp/profile.json".format(odahu_conf['bastion_tag']))
- local("sed -i \'s|<PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['project_id']))
- local("sed -i \'s|<CLUSTER_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_cluster_name']))
- local("sed -i \'s|<REGION>|{}|g\' /tmp/profile.json".format(odahu_conf['region']))
- local("sed -i \'s|<ZONE>|{}|g\' /tmp/profile.json".format(odahu_conf['zone']))
- local("sed -i \'s|<DNS_ZONE_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_zone_name']))
- local("sed -i \'s|<DOCKER_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['docker_repo']))
- local("sed -i \'s|<ODAHU_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_cidr']))
- local("sed -i \'s|<GRAFANA_ADMIN>|{}|g\' /tmp/profile.json".format(odahu_conf['grafana_admin']))
- local("sed -i \'s|<GRAFANA_PASS>|{}|g\' /tmp/profile.json".format(odahu_conf['grafana_pass']))
- local("sed -i \'s|<INITIAL_NODE_COUNT>|{}|g\' /tmp/profile.json".format(odahu_conf['initial_node_count']))
- local("sed -i \'s|<ISTIO_HELM_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['istio_helm_repo']))
- local("sed -i \'s|<HELM_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['helm_repo']))
- local("sed -i \'s|<K8S_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['k8s_version']))
- local("sed -i \'s|<ODAHU_INFRA_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_infra_version']))
- local("sed -i \'s|<ODAHUFLOW_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['odahuflow_version']))
- local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\' /tmp/profile.json".format(
- odahu_conf['mlflow_toolchain_version']))
- local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['jupyterlab_version']))
- local("sed -i \'s|<PACKAGER_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['packager_version']))
- local("sed -i \'s|<MODEL_DOCKER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['model_docker_url']))
- local("sed -i \'s|<NODE_LOCATIONS>|{}|g\' /tmp/profile.json".format(odahu_conf['node_locations']))
- local("sed -i \'s|<NODE_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['node_version']))
- local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
- local("sed -i \'s|<VPC_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['vpc_name']))
- local("sed -i \'s|<SUBNET_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['private_subnet_name']))
- local("sed -i \'s|<OAUTH_CLIENT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_client_id']))
- local("sed -i \'s|<OAUTH_CLIENT_SECRET>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_client_secret']))
- local("sed -i \'s|<OAUTH_COOCKIE_SECRET>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_cookie_secret']))
- local("sed -i \'s|<PODS_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['pods_cidr']))
- local("sed -i \'s|<ROOT_DOMAIN>|{}|g\' /tmp/profile.json".format(odahu_conf['root_domain']))
- local("sed -i \'s|<SERVICE_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['service_cidr']))
- local("sed -i \'s|<TLS_CRT>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
- local("sed -i \'s|<TLS_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
- local("sed -i \'s|<SSH_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
- local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_project_id']))
- local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\' /tmp/profile.json".format(odahu_conf['decrypt_token']))
- local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\' /tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {"allowed_ips": odahu_conf['allowed_cidr'],
+ "cloud_type": "gcp",
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "gcp/gke",
+ "dns_zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ "docker_password": "",
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_user": "",
+ "gcp_cidr": "{}".format(odahu_conf['odahu_cidr']),
+ "gke_node_tag": "{}-gke-node".format(odahu_conf['cluster_name']),
+ "grafana_admin": "{}".format(odahu_conf['grafana_admin']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "odahu_infra_version": "{}".format(odahu_conf['odahu_infra_version']),
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "node_locations": odahu_conf['node_locations'],
+ "node_labels": {
+ "name": "{}".format(odahu_conf['cluster_name']),
+ "product": "dlab",
+ "user": "{}".format(odahu_conf['edge_user_name']),
+ "sbn": "{}".format(odahu_conf['service_base_name'])
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_jwks_url": "https://idp.demo.dlabanalytics.com/auth/realms/dlab/protocol/openid-connect/certs",
+ "oauth_oidc_port": 443,
+ "oauth_local_jwks": "",
+ "oauth_mesh_enabled": "false",
+ "pods_cidr": "{}".format(odahu_conf['pods_cidr']),
+ "project_id": "{}".format(odahu_conf['project_id']),
+ "region": "{}".format(odahu_conf['region']),
+ "root_domain": "{}".format(odahu_conf['root_domain']),
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "zone": "{}".format(odahu_conf['zone']),
+ "dns_project_id": "{}".format(odahu_conf['dns_project_id']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "authorization_enabled": "true",
+ "authz_dry_run": "true",
+ "k8s_version": "1.13.12-gke.25",
+ "node_pools": {
+ "packaging": {
+ "disk_size_gb": 64,
+ "max_node_count": 3,
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "taints": [
+ {
+ "value": "packaging",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-standard-4",
+ "disk_type": "pd-ssd"
+ },
+ "training": {
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "disk_size_gb": 100,
+ "taints": [
+ {
+ "value": "training",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-highcpu-8"
+ },
+ "main": {
+ "init_node_count": 3,
+ "disk_size_gb": 64,
+ "min_node_count": 1,
+ "max_node_count": 5
+ },
+ "training_gpu": {
+ "labels": {
+ "mode": "odahu-flow-training-gpu"
+ },
+ "gpu": [
+ {
+ "count": 2,
+ "type": "nvidia-tesla-p100"
+ }
+ ],
+ "disk_size_gb": 100,
+ "taints": [
+ {
+ "value": "training-gpu",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-standard-8"
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "taints": [
+ {
+ "value": "deployment",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "max_node_count": 3
+ }
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
local('cp /tmp/profile.json /')
- local('cat /profile.json')
except Exception as err:
traceback.print_exc()
append_result("Failed to configure parameter file.", str(err))
@@ -146,5 +233,5 @@ if __name__ == "__main__":
local('tf_runner suspend -v')
except Exception as err:
traceback.print_exc()
- append_result("Failed to deploy Odahu cluster.", str(err))
+ append_result("Failed to suspend Odahu cluster.", str(err))
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
index 23dc9a5..9b0f1f0 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
@@ -28,6 +28,7 @@ from dlab.fab import *
from dlab.meta_lib import *
from dlab.actions_lib import *
import os
+import base64
if __name__ == "__main__":
@@ -38,16 +39,13 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
- secret = ''.join(random.choices(string.ascii_uppercase + +string.ascii_lowercase +
- string.digits, k=16))
-
print('Generating infrastructure names and tags')
odahu_conf = dict()
- odahu_conf['allowed_cidr'] = json.dumps(os.environ['odahu_allowed_cidr'].split(','))
- odahu_conf['bastion_tag'] = os.environ['odahu_bastion_tag']
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
odahu_conf['project_id'] = (os.environ['gcp_project_id'])
odahu_conf['region'] = (os.environ['gcp_region'])
odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['node_locations'] = os.environ['odahu_node_locations'].split(',')
odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
@@ -67,78 +65,164 @@ if __name__ == "__main__":
odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
odahu_conf['project_name'])
odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
- odahu_conf['grafana_pass'] = os.environ['odahu_grafana_pass']
+ odahu_conf['grafana_pass'] = id_generator()
odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
- odahu_conf['oauth_oidc_issuer_url'] = os.environ['oauth_oidc_issuer_url']
- odahu_conf['oauth_client_id'] = os.environ['oauth_client_id']
- odahu_conf['oauth_client_secret'] = os.environ['oauth_client_secret']
- odahu_conf['oauth_cookie_secret'] = secret
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['odahu_oauth_oidc_host']
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = base64.b64encode(id_generator(16))
odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
odahu_conf['packager_version'] = os.environ['odahu_packager_version']
- odahu_conf['model_docker_url'] = os.environ['odahu_model_docker_url']
- odahu_conf['node_locations'] = json.dumps(os.environ['odahu_node_locations'].split(','))
odahu_conf['node_version'] = os.environ['odahu_node_version']
odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
odahu_conf['root_domain'] = os.environ['odahu_root_domain']
odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
- odahu_conf['tls_crt'] = base64.b64decode(os.environ['tls_crt'])
- odahu_conf['tls_key'] = base64.b64decode(os.environ['tls_key'])
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
odahu_conf['ssh_key'] = os.environ['ssh_key']
- odahu_conf['dns_project_id'] = os.environ['dns_project_id']
- odahu_conf['decrypt_token'] = secret
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = id_generator()
odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
print('Preparing parameters file')
try:
- print(odahu_conf['allowed_cidr'])
local("cp /root/templates/profile.json /tmp/")
- local("sed -i \'s|<ALLOWED_IP_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['allowed_cidr']))
- local("sed -i \'s|<BASTION_TAG>|{}|g\' /tmp/profile.json".format(odahu_conf['bastion_tag']))
- local("sed -i \'s|<PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['project_id']))
- local("sed -i \'s|<CLUSTER_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_cluster_name']))
- local("sed -i \'s|<REGION>|{}|g\' /tmp/profile.json".format(odahu_conf['region']))
- local("sed -i \'s|<ZONE>|{}|g\' /tmp/profile.json".format(odahu_conf['zone']))
- local("sed -i \'s|<DNS_ZONE_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_zone_name']))
- local("sed -i \'s|<DOCKER_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['docker_repo']))
- local("sed -i \'s|<ODAHU_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_cidr']))
- local("sed -i \'s|<GRAFANA_ADMIN>|{}|g\' /tmp/profile.json".format(odahu_conf['grafana_admin']))
- local("sed -i \'s|<GRAFANA_PASS>|{}|g\' /tmАp/profile.json".format(odahu_conf['grafana_pass']))
- local("sed -i \'s|<INITIAL_NODE_COUNT>|{}|g\' /tmp/profile.json".format(odahu_conf['initial_node_count']))
- local("sed -i \'s|<ISTIO_HELM_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['istio_helm_repo']))
- local("sed -i \'s|<HELM_REPO>|{}|g\' /tmp/profile.json".format(odahu_conf['helm_repo']))
- local("sed -i \'s|<K8S_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['k8s_version']))
- local("sed -i \'s|<ODAHU_INFRA_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['odahu_infra_version']))
- local("sed -i \'s|<ODAHUFLOW_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['odahuflow_version']))
- local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\' /tmp/profile.json".format(
- odahu_conf['mlflow_toolchain_version']))
- local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['jupyterlab_version']))
- local("sed -i \'s|<PACKAGER_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['packager_version']))
- local("sed -i \'s|<MODEL_DOCKER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['model_docker_url']))
- local("sed -i \'s|<NODE_LOCATIONS>|{}|g\' /tmp/profile.json".format(odahu_conf['node_locations']))
- local("sed -i \'s|<NODE_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['node_version']))
- local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
- local("sed -i \'s|<VPC_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['vpc_name']))
- local("sed -i \'s|<SUBNET_NAME>|{}|g\' /tmp/profile.json".format(odahu_conf['private_subnet_name']))
- local("sed -i \'s|<OAUTH_CLIENT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_client_id']))
- local("sed -i \'s|<OAUTH_CLIENT_SECRET>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_client_secret']))
- local("sed -i \'s|<OAUTH_COOCKIE_SECRET>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_cookie_secret']))
- local("sed -i \'s|<PODS_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['pods_cidr']))
- local("sed -i \'s|<ROOT_DOMAIN>|{}|g\' /tmp/profile.json".format(odahu_conf['root_domain']))
- local("sed -i \'s|<SERVICE_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['service_cidr']))
- local("sed -i \'s|<TLS_CRT>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
- local("sed -i \'s|<TLS_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
- local("sed -i \'s|<SSH_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
- local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_project_id']))
- local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\' /tmp/profile.json".format(odahu_conf['decrypt_token']))
- local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\' /tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {"allowed_ips": odahu_conf['allowed_cidr'],
+ "cloud_type": "gcp",
+ "cluster_name": "{}".format(odahu_conf['odahu_cluster_name']),
+ "cluster_type": "gcp/gke",
+ "cluster_context": "",
+ "dns_zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ "docker_password": "",
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_user": "",
+ "gcp_cidr": "{}".format(odahu_conf['odahu_cidr']),
+ "gke_node_tag": "{}-gke-node".format(odahu_conf['odahu_cluster_name']).split(','),
+ "grafana_admin": "{}".format(odahu_conf['grafana_admin']),
+ "grafana_pass": "BMuYxTw6v5",
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "data_bucket": "{}-data-bucket".format(odahu_conf['odahu_cluster_name']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "odahu_infra_version": "{}".format(odahu_conf['odahu_infra_version']),
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "node_locations": odahu_conf['node_locations'],
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "OWl6VnV3SHNIQzVyMThqVw==",
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_jwks_url": "https://idp.demo.dlabanalytics.com/auth/realms/dlab/protocol/openid-connect/certs",
+ "oauth_oidc_port": 443,
+ "pods_cidr": "{}".format(odahu_conf['pods_cidr']),
+ "project_id": "{}".format(odahu_conf['project_id']),
+ "region": "{}".format(odahu_conf['region']),
+ "root_domain": "{}".format(odahu_conf['root_domain']),
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['odahu_cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "zone": "{}".format(odahu_conf['zone']),
+ "dns_project_id": "{}".format(odahu_conf['dns_project_id']),
+ "odahuflow_connection_decrypt_token": "qfY2k5bA2M",
+ "authorization_enabled": "true",
+ "authz_dry_run": "true",
+ "github_org_name": "",
+ "oauth_local_jwks": "",
+ "oauth_mesh_enabled": "false",
+ "k8s_version": "1.14.10-gke.17",
+ "opa_policies": {
+ "policy.rego": "cGFja2FnZSBvZGFodQoKIyByb2xlcy5yZWdvCgphZG1pbiA6PSAiYWRtaW4iCmRhdGFfc2NpZW50aXN0IDo9ICJkYXRhX3NjaWVudGlzdCIKCiMgcmJhYy5yZWdvCgpyYmFjIDo9IHsKCWRhdGFfc2NpZW50aXN0OiBbCiAgICAJWyIqIiwgImFwaS92MS9tb2RlbC9kZXBsb3ltZW50LioiXSwKICAgIAlbIioiLCAiYXBpL3YxL21vZGVsL3BhY2thZ2luZy4qIl0sCiAgICAJWyIqIiwgImFwaS92MS9tb2RlbC90cmFpbmluZy4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3YxL2Nvbm5lY3Rpb24uKiJdLAogICAgCVsiUE9TVCIsICJhcGkvdjEvY29ubmVjdGlvbi4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3Yx [...]
+ "node_pools": {
+ "packaging": {
+ "disk_size_gb": 64,
+ "max_node_count": 3,
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "taints": [
+ {
+ "value": "packaging",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-standard-4",
+ "disk_type": "pd-ssd"
+ },
+ "training": {
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "disk_size_gb": 100,
+ "taints": [
+ {
+ "value": "training",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-highcpu-8"
+ },
+ "main": {
+ "init_node_count": 3,
+ "disk_size_gb": 64,
+ "min_node_count": 1,
+ "max_node_count": 5
+ },
+ "training_gpu": {
+ "labels": {
+ "mode": "odahu-flow-training-gpu"
+ },
+ "gpu": [
+ {
+ "count": 2,
+ "type": "nvidia-tesla-p100"
+ }
+ ],
+ "disk_size_gb": 100,
+ "taints": [
+ {
+ "value": "training-gpu",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "machine_type": "n1-standard-8"
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "taints": [
+ {
+ "value": "deployment",
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated"
+ }
+ ],
+ "max_node_count": 3
+ }
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
local('cp /tmp/profile.json /')
- local('cat /profile.json')
except Exception as err:
traceback.print_exc()
append_result("Failed to configure parameter file.", str(err))
diff --git a/infrastructure-provisioning/src/odahu/fabfile.py b/infrastructure-provisioning/src/odahu/fabfile.py
index 19beb83..85b09fb 100644
--- a/infrastructure-provisioning/src/odahu/fabfile.py
+++ b/infrastructure-provisioning/src/odahu/fabfile.py
@@ -39,20 +39,15 @@ def run():
level=logging.DEBUG,
filename=local_log_filepath)
- notebook_config = dict()
- notebook_config['uuid'] = str(uuid.uuid4())[:5]
-
try:
- params = "--uuid {}".format(notebook_config['uuid'])
- local("~/scripts/{}.py {}".format('odahu_prepare', params))
+ local("~/scripts/{}.py".format('odahu_prepare'))
except Exception as err:
traceback.print_exc()
append_result("Failed preparing Notebook node.", str(err))
sys.exit(1)
try:
- params = "--uuid {}".format(notebook_config['uuid'])
- local("~/scripts/{}.py {}".format('odahu_deploy', params))
+ local("~/scripts/{}.py".format('odahu_deploy'))
except Exception as err:
traceback.print_exc()
append_result("Failed to deploy Odahuflow cluster.", str(err))
@@ -81,8 +76,7 @@ def stop():
level=logging.DEBUG,
filename=local_log_filepath)
try:
- params = "--uuid {}".format(notebook_config['uuid'])
- local("~/scripts/{}.py {}".format('odahu_suspend', params))
+ local("~/scripts/{}.py".format('odahu_suspend'))
except Exception as err:
traceback.print_exc()
append_result("Failed to suspend Odahuflow cluster.", str(err))
@@ -96,9 +90,8 @@ def start():
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
filename=local_log_filepath)
-
try:
- local("tf_runner resume")
+ local("~/scripts/{}.py".format('odahu_resume'))
except Exception as err:
traceback.print_exc()
append_result("Failed to resume Odahuflow cluster.", str(err))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org