You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by my...@apache.org on 2020/02/05 10:50:44 UTC
[incubator-dlab] 03/04: [DLAB-1430]: Post-deployment configuration
scripts for SSN prepared
This is an automated email from the ASF dual-hosted git repository.
mykolabodnar pushed a commit to branch DLAB-1430
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 9b7f10bf8c72852712165ad99fd58e6e61292e95
Author: Mykola_Bodnar1 <bo...@gmail.com>
AuthorDate: Wed Feb 5 12:46:33 2020 +0200
[DLAB-1430]: Post-deployment configuration scripts for SSN prepared
---
.../scripts/POST_DEPLOYMENT.md | 42 ++++++++++
...uration.py => post-deployment_configuration.py} | 96 +++++++++++++++++-----
.../scripts/post_deployment_configuration.sh | 56 -------------
.../src/ssn/scripts/docker_build.py | 4 +-
4 files changed, 119 insertions(+), 79 deletions(-)
diff --git a/infrastructure-provisioning/scripts/POST_DEPLOYMENT.md b/infrastructure-provisioning/scripts/POST_DEPLOYMENT.md
new file mode 100644
index 0000000..42d826f
--- /dev/null
+++ b/infrastructure-provisioning/scripts/POST_DEPLOYMENT.md
@@ -0,0 +1,42 @@
+### Prerequisites for DLab post-deployment
+
+- Service account with following roles:
+```
+Compute Admin
+Compute Network Admin
+Dataproc Administrator
+Role Administrator
+Service Account Admin
+Service Account User
+Project IAM Admin
+Storage Admin
+```
+- Google Cloud Storage JSON API should be enabled
+- Keycloak server with specific client for Dlab UI (could be dpeloyed with Kecylaok deployment script)
+
+Service account should be created manually and attached to the instance with post-deployment script.
+
+### Executing post-deployment script
+
+To configure SSN node, following steps should be executed:
+
+- Connect to the instance via SSH and run the following commands:
+```
+/usr/bin/python /opt/dlab/sources/infrastructure-provisioning/scripts/post-deployment_configuration.py
+ --keycloak_realm_name <value>
+ --keycloak_auth_server_url <value>
+ --keycloak_client_name <value>
+ --keycloak_client_secret <value>
+ --keycloak_user <value>
+ --keycloak_admin_password <value>
+```
+
+List of parameters for SSN node post-deployment script:
+| Parameter | Description/Value |
+|-------------------------------|-------------------------------------------------------------------------------------|
+| keycloak\_realm\_name | Keycloak realm name |
+| keycloak\_auth\_server\_url | Url of Keycloak auth server |
+| keycloak\_client\_name | Name of client for Dlab UI |
+| keycloak\_client\_secret | Secret of client for Dlab UI |
+| kkeycloak\_user | Keycloak user with administrator permissions |
+| keycloak\_admin\_password | Password for Keycloak user with administrator permissions |
\ No newline at end of file
diff --git a/infrastructure-provisioning/scripts/post_deployment_configuration.py b/infrastructure-provisioning/scripts/post-deployment_configuration.py
similarity index 60%
rename from infrastructure-provisioning/scripts/post_deployment_configuration.py
rename to infrastructure-provisioning/scripts/post-deployment_configuration.py
index 5e5271e..2be6807 100644
--- a/infrastructure-provisioning/scripts/post_deployment_configuration.py
+++ b/infrastructure-provisioning/scripts/post-deployment_configuration.py
@@ -24,17 +24,19 @@
from fabric.api import *
import argparse
import requests
+import uuid
+from Crypto.PublicKey import RSA
if __name__ == "__main__":
parser = argparse.ArgumentParser()
- parser.add_argument('--keycloak_realm_name', type=str, default='dlab', help='Keycloak Realm name')
- parser.add_argument('--keycloak_auth_server_url', type=str, default='dlab', help='Keycloak auth server URL')
- parser.add_argument('--keycloak_client_name', type=str, default='dlab', help='Keycloak client name')
- parser.add_argument('--keycloak_client_secret', type=str, default='dlab', help='Keycloak client secret')
- parser.add_argument('--keycloak_user', type=str, default='dlab', help='Keycloak user')
- parser.add_argument('--keycloak_user_password', type=str, default='keycloak-user-password',
- help='Keycloak user password')
+ parser.add_argument('--keycloak_realm_name', type=str, default='KEYCLOAK_REALM_NAME', help='Keycloak Realm name')
+ parser.add_argument('--keycloak_auth_server_url', type=str, default='KEYCLOAK_AUTH_SERVER_URL', help='Keycloak auth server URL')
+ parser.add_argument('--keycloak_client_name', type=str, default='KEYCLOAK_CLIENT_NAME', help='Keycloak client name')
+ parser.add_argument('--keycloak_client_secret', type=str, default='KEYCLOAK_CLIENT_SECRET', help='Keycloak client secret')
+ parser.add_argument('--keycloak_user', type=str, default='KEYCLOAK_USER', help='Keycloak user')
+ parser.add_argument('--keycloak_admin_password', type=str, default='KEYCLOAK_ADMIN_PASSWORD',
+ help='Keycloak admin password')
args = parser.parse_args()
headers = {
'Metadata-Flavor': 'Google',
@@ -51,6 +53,25 @@ if __name__ == "__main__":
gcp_projectId = requests.get('http://metadata/computeMetadata/v1/project/project-id', headers=headers).text
keycloak_redirectUri = 'http://{}'.format(server_external_ip)
+ print("Generationg SSH keyfile for dlab-user")
+ key = RSA.generate(2048)
+ local("sudo sh -c 'echo \"{}\" >> /home/dlab-user/keys/KEY-FILE.pem'".format(key.exportKey('PEM')))
+ local("sudo chmod 600 /home/dlab-user/keys/KEY-FILE.pem")
+ pubkey = key.publickey()
+ local("sudo sh -c 'echo \"{}\" >> /home/dlab-user/.ssh/authorized_keys'".format(pubkey.exportKey('OpenSSH')))
+
+ print("Generationg MongoDB password")
+ mongo_pwd = uuid.uuid4().hex
+ try:
+ local("sudo echo -e 'db.changeUserPassword(\"admin\", \"{}\")' | mongo dlabdb --port 27017 -u admin -p MONGO_PASSWORD".format(
+ mongo_pwd))
+ local('sudo sed -i "s|MONGO_PASSWORD|{}|g" /opt/dlab/conf/billing.yml'.format(mongo_pwd))
+
+ local('sudo sed -i "s|MONGO_PASSWORD|{}|g" /opt/dlab/conf/ssn.yml'.format(mongo_pwd))
+ except:
+ print('Mongo password was already changed')
+
+
print('Reserving external IP')
static_address_exist = local(
"sudo gcloud compute addresses list --filter='address={}'".format(server_external_ip), capture=True)
@@ -68,13 +89,15 @@ if __name__ == "__main__":
local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/conf/self-service.yml'.format(
args.keycloak_auth_server_url))
local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/conf/self-service.yml'.format(args.keycloak_client_name))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/self-service.yml'.format(args.keycloak_client_secret))
+ local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/self-service.yml'.format(
+ args.keycloak_client_secret))
local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_realm_name))
local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/conf/provisioning.yml'.format(
args.keycloak_auth_server_url))
local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_client_name))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_client_secret))
+ local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/conf/provisioning.yml'.format(
+ args.keycloak_client_secret))
local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/conf/provisioning.yml'.format(dlab_sbn))
local('sudo sed -i "s|SUBNET_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(deployment_subnetId))
local('sudo sed -i "s|DLAB_REGION|{}|g" /opt/dlab/conf/provisioning.yml'.format(dlab_region))
@@ -82,20 +105,49 @@ if __name__ == "__main__":
local('sudo sed -i "s|SSN_VPC_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(deployment_vpcId))
local('sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/dlab/conf/provisioning.yml'.format(gcp_projectId))
local('sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_user))
- local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/conf/provisioning.yml'.format(args.keycloak_user_password))
-
- local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_sbn))
- local('sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(gcp_projectId))
- local('sudo sed -i "s|DLAB_REGION|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_region))
- local('sudo sed -i "s|DLAB_ZONE|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(dlab_zone))
- local('sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_realm_name))
- local('sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_auth_server_url))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_client_name))
- local('sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_client_secret))
- local('sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_user))
- local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(args.keycloak_user_password))
+ local('sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/conf/provisioning.yml'.format(
+ args.keycloak_admin_password))
+
+ local('sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/conf/billing.yml'.format(dlab_sbn))
+
+ local(
+ 'sudo sed -i "s|DLAB_SBN|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ dlab_sbn))
+ local(
+ 'sudo sed -i "s|GCP_PROJECT_ID|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ gcp_projectId))
+ local(
+ 'sudo sed -i "s|DLAB_REGION|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ dlab_region))
+ local(
+ 'sudo sed -i "s|DLAB_ZONE|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ dlab_zone))
+ local(
+ 'sudo sed -i "s|KEYCLOAK_REALM_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ args.keycloak_realm_name))
+ local(
+ 'sudo sed -i "s|KEYCLOAK_AUTH_SERVER_URL|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ args.keycloak_auth_server_url))
+ local(
+ 'sudo sed -i "s|KEYCLOAK_CLIENT_NAME|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ args.keycloak_client_name))
+ local(
+ 'sudo sed -i "s|KEYCLOAK_CLIENT_SECRET|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ args.keycloak_client_secret))
+ local(
+ 'sudo sed -i "s|KEYCLOAK_USER|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ args.keycloak_user))
+ local(
+ 'sudo sed -i "s|KEYCLOAK_ADMIN_PASSWORD|{}|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini'.format(
+ args.keycloak_admin_password))
local('sudo sed -i "s|SERVER_IP|{}|g" /etc/nginx/conf.d/nginx_proxy.conf'.format(server_external_ip))
local('sudo systemctl restart nginx')
local('sudo supervisorctl restart all')
- local('cd /opt/dlab/sources/infrastructure-provisioning/src/ && docker-build all')
\ No newline at end of file
+ local('cd /opt/dlab/sources/infrastructure-provisioning/src/ && sudo docker-build all')
+
+ print('SUMMARY')
+ print('Mongo password stored in /opt/dlab/conf/ssn.yml')
+ print('SSH key for dlab-user stored in /home/dlab-user/keys/KEY-FILE.pem')
+ if not args:
+ print('Keycloak parameters was not set, please configure Keycloak parameters manually')
diff --git a/infrastructure-provisioning/scripts/post_deployment_configuration.sh b/infrastructure-provisioning/scripts/post_deployment_configuration.sh
deleted file mode 100644
index 234e108..0000000
--- a/infrastructure-provisioning/scripts/post_deployment_configuration.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-
-server_external_ip=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
-sed -i "s|SERVER_IP|$server_external_ip|g" /etc/nginx/conf.d/nginx_proxy.conf
-systemctl restart nginx
-
-dlab_sbn=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/name)
-
-KEYCLOAK_REDIRECTURI='http://'$server_external_ip
-KEYCLOAK_REALM_NAME='dlab'
-KEYCLOAK_AUTH_SERVER_URL='https://idp.demo.dlabanalytics.com/auth'
-KEYCLOAK_CLIENT_NAME=$dlab_sbn'-ui'
-KEYCLOAK_CLIENT_SECRET='e235f2b6-a5e0-448a-837d-465d1a4990f7'
-KEYCLOAK_USER='admin'
-KEYCLOAK_USER_PASSWORD='v7rdj2ckHgAdJj54'
-
-sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_REDIRECTURI|$KEYCLOAK_REDIRECTURI|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/self-service.yml
-sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/conf/provisioning.yml
-
-ssn_subnetId=$(sudo gcloud compute instances describe $dlab_sbn --zone us-west1-a | awk -F/ '/subnetwork: / {print $11}')
-dlab_zone=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/zone | awk -F/ '{print $4}')
-dlab_region=$(echo $dlab_zone | awk '{print substr($0, 1, length($0)-2)}')
-ssn_vpcId=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/network | awk -F/ '{print $4}')
-gcp_projectId=$(curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/project/project-id)
-
-sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|SUBNET_ID|$ssn_subnetId|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|DLAB_REGION|$dlab_region|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|DLAB_ZONE|$dlab_zone|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|SSN_VPC_ID|$ssn_vpcId|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|GCP_PROJECT_ID|$gcp_projectId|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|KEYCLOAK_USER|$KEYCLOAK_USER|g" /opt/dlab/conf/provisioning.yml
-sed -i "s|KEYCLOAK_USER_PASSWORD|$KEYCLOAK_USER_PASSWORD|g" /opt/dlab/conf/provisioning.yml
-
-sed -i "s|DLAB_SBN|$dlab_sbn|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|GCP_PROJECT_ID|$gcp_projectId|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|DLAB_REGION|$dlab_region|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|DLAB_ZONE|$dlab_zone|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|KEYCLOAK_REALM_NAME|$KEYCLOAK_REALM_NAME|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|KEYCLOAK_AUTH_SERVER_URL|$KEYCLOAK_AUTH_SERVER_URL|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|KEYCLOAK_CLIENT_NAME|$KEYCLOAK_CLIENT_NAME|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|KEYCLOAK_CLIENT_SECRET|$KEYCLOAK_CLIENT_SECRET|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|KEYCLOAK_USER|$KEYCLOAK_USER|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-sed -i "s|KEYCLOAK_USER_PASSWORD|$KEYCLOAK_USER_PASSWORD|g" /opt/dlab/sources/infrastructure-provisioning/src/general/conf/overwrite.ini
-
-supervisorctl restart all
-
-cd /opt/dlab/sources/infrastructure-provisioning/src/ && docker-build all
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/ssn/scripts/docker_build.py b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
index 73b5a1d..ac4fee5 100644
--- a/infrastructure-provisioning/src/ssn/scripts/docker_build.py
+++ b/infrastructure-provisioning/src/ssn/scripts/docker_build.py
@@ -40,7 +40,9 @@ if sys.argv[1] == 'all':
'tensor',
'tensor-rstudio',
'deeplearning',
- 'dataengine'
+ 'dataengine',
+ 'dataengine-service',
+ 'superset'
]
else:
node = sys.argv[1:]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org