You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by dm...@apache.org on 2020/02/06 12:04:05 UTC
[incubator-dlab] branch odahu-integration updated:
[odahu-integration] Bugfixing and added function that gets available zones
This is an automated email from the ASF dual-hosted git repository.
dmysakovets pushed a commit to branch odahu-integration
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/odahu-integration by this push:
new a43330c [odahu-integration] Bugfixing and added function that gets available zones
new f6e8de6 Merge remote-tracking branch 'origin/odahu-integration' into odahu-integration
a43330c is described below
commit a43330cd8d1955abe1044e394b480433d8025eac
Author: Demyan Mysakovets <de...@gmail.com>
AuthorDate: Thu Feb 6 14:03:09 2020 +0200
[odahu-integration] Bugfixing and added function that gets available zones
---
.../src/general/conf/dlab.ini | 2 ++
.../src/general/files/gcp/odahu_Dockerfile | 6 ++++--
.../src/general/files/gcp/ssn_policy.json | 18 +++++++++++++++++-
.../src/general/files/gcp/ssn_roles.json | 3 ++-
.../src/general/lib/gcp/meta_lib.py | 19 +++++++++++++++++--
.../src/general/scripts/gcp/odahu_deploy.py | 22 ++++++++++------------
.../src/odahu/templates/profile.json | 1 -
7 files changed, 52 insertions(+), 19 deletions(-)
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index d1f8731..83e0c5a 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -321,6 +321,8 @@ expl_instance_memory = 8000
#--- [odahu] odahuflow parameters ---#
[odahu]
###
+allowed_cidr = 0.0.0.0/0
+###
bastion_tag = odahubastion
###
dns_zone_name = ailifecycle-org
diff --git a/infrastructure-provisioning/src/general/files/gcp/odahu_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/odahu_Dockerfile
index c360ab8..1374163 100644
--- a/infrastructure-provisioning/src/general/files/gcp/odahu_Dockerfile
+++ b/infrastructure-provisioning/src/general/files/gcp/odahu_Dockerfile
@@ -23,6 +23,8 @@ FROM <NEXUS_URL>/odahu-flow-automation:1.1.0
ARG OS
ARG SRC_PATH
+ENV PROFILE=/profile.json
+
# Install any .deb dependecies
RUN apt-get update && \
apt-get -y upgrade && \
@@ -31,8 +33,8 @@ RUN apt-get update && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Install any python dependencies
-RUN pip install -UI pip==9.0.3 && \
- pip install boto3 backoff fabric==1.14.0 fabvenv argparse ujson pycrypto google-api-python-client google-cloud-storage \
+RUN pip2 install -UI pip==9.0.3 && \
+ pip2 install boto3 backoff fabric==1.14.0 fabvenv argparse ujson pycrypto google-api-python-client google-cloud-storage \
pyyaml google-auth-httplib2 oauth2client
# Configuring ssh for user
diff --git a/infrastructure-provisioning/src/general/files/gcp/ssn_policy.json b/infrastructure-provisioning/src/general/files/gcp/ssn_policy.json
index bd95d12..35e8307 100644
--- a/infrastructure-provisioning/src/general/files/gcp/ssn_policy.json
+++ b/infrastructure-provisioning/src/general/files/gcp/ssn_policy.json
@@ -16,5 +16,21 @@
"compute.images.get",
"compute.images.delete",
"compute.images.setLabels",
- "compute.images.list"
+ "compute.images.list",
+ "container.clusters.create",
+ "container.clusters.delete",
+ "container.clusters.get",
+ "container.clusters.update",
+ "container.operations.get",
+ "compute.routers.create",
+ "compute.routers.delete",
+ "compute.routers.get",
+ "compute.routers.update",
+ "container.serviceAccounts.create",
+ "container.serviceAccounts.get",
+ "container.serviceAccounts.delete",
+ "iam.serviceAccountKeys.create",
+ "iam.serviceAccountKeys.delete",
+ "dns.resourceRecordSets.list",
+ "dns.managedZones.get"
]
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/files/gcp/ssn_roles.json b/infrastructure-provisioning/src/general/files/gcp/ssn_roles.json
index fe310b6..4da449f 100644
--- a/infrastructure-provisioning/src/general/files/gcp/ssn_roles.json
+++ b/infrastructure-provisioning/src/general/files/gcp/ssn_roles.json
@@ -7,5 +7,6 @@
"iam.roleAdmin",
"compute.instanceAdmin",
"bigquery.dataViewer",
- "bigquery.jobUser"
+ "bigquery.jobUser",
+ "container.admin"
]
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
index 5e7a96b..bc9905d 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
@@ -40,7 +40,6 @@ class GCPMeta:
google.auth.exceptions.DefaultCredentialsError,
max_tries=15)
def get_gcp_cred():
- credentials, project = google.auth.default()
return credentials, project
self.auth_type = auth_type
@@ -54,7 +53,6 @@ class GCPMeta:
['https://www.googleapis.com/auth/compute',
'https://www.googleapis.com/auth/iam',
'https://www.googleapis.com/auth/cloud-platform'])
- self.service = build('compute', 'v1', credentials=credentials)
self.service_iam = build('iam', 'v1', credentials=credentials)
self.dataproc = build('dataproc', 'v1', credentials=credentials)
self.service_storage = build('storage', 'v1', credentials=credentials)
@@ -698,6 +696,23 @@ class GCPMeta:
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+ def get_available_zones(self):
+ try:
+ request = self.service.regions().get(project=self.project, region=os.environ['gcp_region'])
+ zone_list = []
+ response = request.execute()
+ for zone in response['zones']:
+ zone_list.append(str(zone.split('/')[-1]))
+ return zone_list
+ except Exception as err:
+ logging.info(
+ "Error with getting available zones: " + str(err) + "\n Traceback: " + traceback.print_exc(
+ file=sys.stdout))
+ append_result(str({"error": "Error with getting available zones",
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
+ traceback.print_exc(file=sys.stdout)
+ return ''
+
def get_instance_private_ip_address(tag_name, instance_name):
try:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
index bbdf11a..b5ca560 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
@@ -38,8 +38,7 @@ if __name__ == "__main__":
level=logging.DEBUG,
filename=local_log_filepath)
- secret = ''.join(random.choices(string.ascii_uppercase + +string.ascii_lowercase +
- string.digits, k=16))
+
print('Generating infrastructure names and tags')
odahu_conf = dict()
@@ -48,6 +47,7 @@ if __name__ == "__main__":
odahu_conf['project_id'] = (os.environ['gcp_project_id'])
odahu_conf['region'] = (os.environ['gcp_region'])
odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['node_locations'] = json.dumps(GCPMeta().get_available_zones())
odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
odahu_conf['odahu_cidr'] = os.environ['odahu_cidr']
@@ -75,23 +75,21 @@ if __name__ == "__main__":
odahu_conf['oauth_oidc_issuer_url'] = "{}realms/{}".format(os.environ['keycloak_auth_server_url'], os.environ['keycloak_realm_name'])
odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
- odahu_conf['oauth_cookie_secret'] = secret
+ odahu_conf['oauth_cookie_secret'] = id_generator()
odahu_conf['odahu_infra_version'] = os.environ['odahu_infra_version']
odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
odahu_conf['packager_version'] = os.environ['odahu_packager_version']
- odahu_conf['model_docker_url'] = os.environ['odahu_model_docker_url']
- odahu_conf['node_locations'] = json.dumps(os.environ['odahu_node_locations'].split(','))
odahu_conf['node_version'] = os.environ['odahu_node_version']
odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
odahu_conf['root_domain'] = os.environ['odahu_root_domain']
odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
- odahu_conf['tls_crt'] = base64.b64decode(os.environ['tls_crt'])
- odahu_conf['tls_key'] = base64.b64decode(os.environ['tls_key'])
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
odahu_conf['ssh_key'] = os.environ['ssh_key']
- odahu_conf['dns_project_id'] = os.environ['dns_project_id']
- odahu_conf['decrypt_token'] = secret
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = id_generator()
odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
print('Preparing parameters file')
@@ -117,7 +115,6 @@ if __name__ == "__main__":
local("sed -i \'s|<MLFLOW_TOOLCHAIN_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['mlflow_toolchain_version']))
local("sed -i \'s|<JUPYTERLAB_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['jupyterlab_version']))
local("sed -i \'s|<PACKAGER_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['packager_version']))
- local("sed -i \'s|<MODEL_DOCKER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['model_docker_url']))
local("sed -i \'s|<NODE_LOCATIONS>|{}|g\' /tmp/profile.json".format(odahu_conf['node_locations']))
local("sed -i \'s|<NODE_VERSION>|{}|g\' /tmp/profile.json".format(odahu_conf['node_version']))
local("sed -i \'s|<OAUTH_OIDC_ISSUER_URL>|{}|g\' /tmp/profile.json".format(odahu_conf['oauth_oidc_issuer_url']))
@@ -131,10 +128,11 @@ if __name__ == "__main__":
local("sed -i \'s|<SERVICE_CIDR>|{}|g\' /tmp/profile.json".format(odahu_conf['service_cidr']))
local("sed -i \'s|<TLS_CRT>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_crt'].replace('\n', '')))
local("sed -i \'s|<TLS_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
- local("sed -i \'s|<SSH_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['tls_key'].replace('\n', '')))
+ local("sed -i \'s|<SSH_KEY>|{}|g\' /tmp/profile.json".format(odahu_conf['ssh_key'].replace('\n', '')))
local("sed -i \'s|<DNS_PROJECT_ID>|{}|g\' /tmp/profile.json".format(odahu_conf['dns_project_id']))
local("sed -i \'s|<DECRYPT_TOKEN>|{}|g\' /tmp/profile.json".format(odahu_conf['decrypt_token']))
local("sed -i \'s|<INFRA_VPC_PEERING>|{}|g\' /tmp/profile.json".format(odahu_conf['infra_vpc_peering']))
+ local("sed -i \'s|\r||g\' /tmp/profile.json")
local('cp /tmp/profile.json /')
local('cat /profile.json')
except Exception as err:
@@ -143,7 +141,7 @@ if __name__ == "__main__":
sys.exit(1)
try:
- local('tf_runner create')
+ local('tf_runner create -o /response')
except Exception as err:
traceback.print_exc()
append_result("Failed to deploy Odahu cluster.", str(err))
diff --git a/infrastructure-provisioning/src/odahu/templates/profile.json b/infrastructure-provisioning/src/odahu/templates/profile.json
index 98f93a2..a4b2232 100644
--- a/infrastructure-provisioning/src/odahu/templates/profile.json
+++ b/infrastructure-provisioning/src/odahu/templates/profile.json
@@ -42,7 +42,6 @@
"mlflow_toolchain_version": "<MLFLOW_TOOLCHAIN_VERSION>",
"jupyterlab_version": "<JUPYTERLAB_VERSION>",
"packager_version": "<PACKAGER_VERSION>",
- "model_docker_url": "<MODEL_DOCKER_URL>",
"vpc_name": "<VPC_NAME>",
"network_name": "<VPC_NAME>",
"subnet_name": "<SUBNET_NAME>",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org