You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by ad...@apache.org on 2019/09/25 09:57:39 UTC
[incubator-dlab] branch DLAB-terraform-fixing created (now 33108b4)
This is an automated email from the ASF dual-hosted git repository.
adamsd pushed a change to branch DLAB-terraform-fixing
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git.
at 33108b4 Endpoint provisioning unification.
This branch includes the following new commits:
new 33108b4 Endpoint provisioning unification.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org
[incubator-dlab] 01/01: Endpoint provisioning unification.
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adamsd pushed a commit to branch DLAB-terraform-fixing
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 33108b479af1cae3d61b1349d4549a789b689205
Author: AdamsDisturber <ad...@gmail.com>
AuthorDate: Tue Sep 24 16:51:28 2019 +0300
Endpoint provisioning unification.
---
.../terraform/bin/deploy/daemon.json | 6 +-
.../terraform/bin/deploy/endpoint_fab.py | 345 ++++++++++++++-------
.../terraform/bin/deploy/provisioning.yml | 40 ++-
.../terraform/gcp/endpoint/main/instance.tf | 5 +-
.../terraform/gcp/endpoint/main/main.tf | 2 +-
.../terraform/gcp/endpoint/main/variables.tf | 6 +-
.../terraform/gcp/endpoint/provisioning.py | 2 +-
.../terraform/gcp/endpoint/provisioning.yml | 32 +-
8 files changed, 295 insertions(+), 143 deletions(-)
diff --git a/infrastructure-provisioning/terraform/bin/deploy/daemon.json b/infrastructure-provisioning/terraform/bin/deploy/daemon.json
index c2932be..b99eac2 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/daemon.json
+++ b/infrastructure-provisioning/terraform/bin/deploy/daemon.json
@@ -1,5 +1,5 @@
{
+ DNS_IP_RESOLVE
"insecure-registries": ["REPOSITORY"],
- "disable-legacy-registry": true,
- "dns": ["DNS_IP_RESOLVE"]
-}
+ "disable-legacy-registry": true
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
index 5602fa1..da4d1da 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
+++ b/infrastructure-provisioning/terraform/bin/deploy/endpoint_fab.py
@@ -22,8 +22,7 @@ def create_user():
conn.sudo('useradd -m -G {1} -s /bin/bash {0}'
.format(args.os_user, sudo_group))
conn.sudo(
- 'bash -c \'echo "{} ALL = NOPASSWD:ALL" >> /etc/sudoers\''
- .format(args.os_user, initial_user))
+ 'bash -c \'echo "{} ALL = NOPASSWD:ALL" >> /etc/sudoers\''.format(args.os_user, initial_user))
conn.sudo('mkdir /home/{}/.ssh'.format(args.os_user))
conn.sudo('chown -R {0}:{0} /home/{1}/.ssh/'
.format(initial_user, args.os_user))
@@ -44,12 +43,10 @@ def create_user():
def copy_keys():
try:
- conn.put(args.pkey, '/tmp/')
- conn.sudo('mv /tmp/{0}.pem /home/{1}/keys/'
- .format(args.key_name, args.os_user))
+ conn.put(args.pkey, '/home/{0}/keys/'.format(args.os_user))
conn.sudo('chown -R {0}:{0} /home/{0}/keys'.format(args.os_user))
except Exception as err:
- logging.error('Failed to copy keys ', str(err))
+ logging.error('Failed to copy admin key: ', str(err))
traceback.print_exc()
sys.exit(1)
@@ -88,8 +85,7 @@ def ensure_logs_endpoint():
def ensure_jre_jdk_endpoint():
try:
- if not exists(conn, '/home/{}/.ensure_dir/jre_jdk_ensured'
- .format(args.os_user)):
+ if not exists(conn, '/home/{}/.ensure_dir/jre_jdk_ensured'.format(args.os_user)):
conn.sudo('apt-get install -y openjdk-8-jre-headless')
conn.sudo('apt-get install -y openjdk-8-jdk-headless')
conn.sudo('touch /home/{}/.ensure_dir/jre_jdk_ensured'
@@ -102,8 +98,7 @@ def ensure_jre_jdk_endpoint():
def ensure_supervisor_endpoint():
try:
- if not exists(conn, '/home/{}/.ensure_dir/superv_ensured'
- .format(args.os_user)):
+ if not exists(conn, '/home/{}/.ensure_dir/superv_ensured'.format(args.os_user)):
conn.sudo('apt-get -y install supervisor')
conn.sudo('update-rc.d supervisor defaults')
conn.sudo('update-rc.d supervisor enable')
@@ -117,8 +112,7 @@ def ensure_supervisor_endpoint():
def ensure_docker_endpoint():
try:
- if not exists(conn, '/home/{}/.ensure_dir/docker_ensured'
- .format(args.os_user)):
+ if not exists(conn, '/home/{}/.ensure_dir/docker_ensured'.format(args.os_user)):
conn.sudo("bash -c "
"'curl -fsSL https://download.docker.com/linux/ubuntu/gpg"
" | apt-key add -'")
@@ -129,11 +123,6 @@ def ensure_docker_endpoint():
conn.sudo('apt-cache policy docker-ce')
conn.sudo('apt-get install -y docker-ce={}'
.format(args.docker_version))
- dns_ip_resolve = (conn.run("systemd-resolve --status "
- "| grep -A 5 'Current Scopes: DNS' "
- "| grep 'DNS Servers:' "
- "| awk '{print $3}'")
- .stdout.rstrip("\n\r"))
if not exists(conn, '{}/tmp'.format(args.dlab_path)):
conn.run('mkdir -p {}/tmp'.format(args.dlab_path))
conn.put('./daemon.json',
@@ -142,8 +131,18 @@ def ensure_docker_endpoint():
.format(args.repository_address,
args.repository_port,
args.dlab_path))
- conn.sudo('sed -i "s|DNS_IP_RESOLVE|{}|g" {}/tmp/daemon.json'
- .format(dns_ip_resolve, args.dlab_path))
+ if args.cloud_provider == "aws":
+ dns_ip_resolve = (conn.run("systemd-resolve --status "
+ "| grep -A 5 'Current Scopes: DNS' "
+ "| grep 'DNS Servers:' "
+ "| awk '{print $3}'")
+ .stdout.rstrip("\n\r"))
+ conn.sudo('sed -i "s|DNS_IP_RESOLVE|\"dns\": [{0}],|g" {1}/tmp/daemon.json'
+ .format(dns_ip_resolve, args.dlab_path))
+ elif args.cloud_provider == "gcp":
+ dns_ip_resolve = ""
+ conn.sudo('sed -i "s|DNS_IP_RESOLVE||g" {1}/tmp/daemon.json'
+ .format(dns_ip_resolve, args.dlab_path))
conn.sudo('mv {}/tmp/daemon.json /etc/docker'
.format(args.dlab_path))
conn.sudo('usermod -a -G docker ' + args.os_user)
@@ -170,24 +169,38 @@ def create_key_dir_endpoint():
def configure_keystore_endpoint(os_user):
try:
- conn.sudo('apt-get install -y awscli')
- if not exists(conn, '/home/' + args.os_user + '/keys/endpoint.keystore.jks'):
- conn.sudo('aws s3 cp s3://{0}/dlab/certs/endpoint/endpoint.keystore.jks '
- '/home/{1}/keys/endpoint.keystore.jks'
- .format(args.ssn_bucket_name, args.os_user))
- if not exists(conn, '/home/' + args.os_user + '/keys/dlab.crt'):
- conn.sudo('aws s3 cp s3://{0}/dlab/certs/endpoint/endpoint.crt'
- ' /home/{1}/keys/endpoint.crt'.format(args.ssn_bucket_name, args.os_user))
- if not exists(conn, '/home/' + args.os_user + '/keys/ssn.crt'):
- conn.sudo('aws s3 cp '
- 's3://{0}/dlab/certs/ssn/ssn.crt /home/{1}/keys/ssn.crt'
- .format(args.ssn_bucket_name, args.os_user))
+ if args.cloud_provider == "aws":
+ conn.sudo('apt-get install -y awscli')
+ if not exists(conn, '/home/' + args.os_user + '/keys/endpoint.keystore.jks'):
+ conn.sudo('aws s3 cp s3://{0}/dlab/certs/endpoint/endpoint.keystore.jks '
+ '/home/{1}/keys/endpoint.keystore.jks'
+ .format(args.ssn_bucket_name, args.os_user))
+ if not exists(conn, '/home/' + args.os_user + '/keys/dlab.crt'):
+ conn.sudo('aws s3 cp s3://{0}/dlab/certs/endpoint/endpoint.crt'
+ ' /home/{1}/keys/endpoint.crt'.format(args.ssn_bucket_name, args.os_user))
+ if not exists(conn, '/home/' + args.os_user + '/keys/ssn.crt'):
+ conn.sudo('aws s3 cp '
+ 's3://{0}/dlab/certs/ssn/ssn.crt /home/{1}/keys/ssn.crt'
+ .format(args.ssn_bucket_name, args.os_user))
+ elif args.cloud_provider == "gcp":
+ if not exists(conn, '/home/' + args.os_user + '/keys/endpoint.keystore.jks'):
+ conn.sudo('gsutil -m cp -r gs://{0}/dlab/certs/endpoint/endpoint.keystore.jks '
+ '/home/{1}/keys/'
+ .format(args.ssn_bucket_name, args.os_user))
+ if not exists(conn, '/home/' + args.os_user + '/keys/dlab.crt'):
+ conn.sudo('gsutil -m cp -r gs://{0}/dlab/certs/endpoint/endpoint.crt'
+ ' /home/{1}/keys/'.format(args.ssn_bucket_name, args.os_user))
+ if not exists(conn, '/home/' + args.os_user + '/keys/ssn.crt'):
+ conn.sudo('gsutil -m cp -r '
+ 'gs://{0}/dlab/certs/ssn/ssn.crt /home/{1}/keys/'
+ .format(args.ssn_bucket_name, args.os_user))
if not exists(conn, '/home/' + args.os_user + '/.ensure_dir/cert_imported'):
conn.sudo('keytool -importcert -trustcacerts -alias dlab -file /home/{0}/keys/endpoint.crt -noprompt \
-storepass changeit -keystore {1}/lib/security/cacerts'.format(os_user, java_home))
conn.sudo('keytool -importcert -trustcacerts -file /home/{0}/keys/ssn.crt -noprompt \
-storepass changeit -keystore {1}/lib/security/cacerts'.format(os_user, java_home))
conn.sudo('touch /home/' + args.os_user + '/.ensure_dir/cert_imported')
+ print("Certificates are imported.")
except Exception as err:
print('Failed to configure Keystore certificates: ', str(err))
traceback.print_exc()
@@ -197,8 +210,7 @@ def configure_keystore_endpoint(os_user):
def configure_supervisor_endpoint():
try:
if not exists(conn,
- '/home/{}/.ensure_dir/configure_supervisor_ensured'
- .format(args.os_user)):
+ '/home/{}/.ensure_dir/configure_supervisor_ensured'.format(args.os_user)):
supervisor_conf = '/etc/supervisor/conf.d/supervisor_svc.conf'
if not exists(conn, '{}/tmp'.format(args.dlab_path)):
conn.run('mkdir -p {}/tmp'.format(args.dlab_path))
@@ -228,14 +240,72 @@ def configure_supervisor_endpoint():
.format(java_home, dlab_conf_dir))
conn.sudo('sed -i "s|CLOUD_PROVIDER|{}|g" {}provisioning.yml'
.format(args.cloud_provider, dlab_conf_dir))
- conn.sudo('sed -i "s|SSN_NLB|{}|g" {}provisioning.yml'
- .format(args.ssn_k8s_nlb_dns_name, dlab_conf_dir))
- conn.sudo('sed -i "s|SSN_ALB|{}|g" {}provisioning.yml'
- .format(args.ssn_k8s_alb_dns_name, dlab_conf_dir))
+
+ conn.sudo('sed -i "s|MONGO_HOST|{}|g" {}provisioning.yml'
+ .format(args.mongo_host, dlab_conf_dir))
+ conn.sudo('sed -i "s|MONGO_PORT|{}|g" {}provisioning.yml'
+ .format(args.mongo_port, dlab_conf_dir))
+ conn.sudo('sed -i "s|SS_HOST|{}|g" {}provisioning.yml'
+ .format(args.ss_host, dlab_conf_dir))
+ conn.sudo('sed -i "s|SS_PORT|{}|g" {}provisioning.yml'
+ .format(args.ss_port, dlab_conf_dir))
+ conn.sudo('sed -i "s|KEYCLOACK_HOST|{}|g" {}provisioning.yml'
+ .format(args.keycloack_host, dlab_conf_dir))
+
conn.sudo('sed -i "s|CLIENT_SECRET|{}|g" {}provisioning.yml'
.format(args.keycloak_client_secret, dlab_conf_dir))
# conn.sudo('sed -i "s|MONGO_PASSWORD|{}|g" {}provisioning.yml'
# .format(args.mongo_password, dlab_conf_dir))
+ conn.sudo('sed -i "s|CONF_OS|{}|g" {}provisioning.yml'
+ .format(args.conf_os, dlab_conf_dir))
+ conn.sudo('sed -i "s|SERVICE_BASE_NAME|{}|g" {}provisioning.yml'
+ .format(args.service_base_name, dlab_conf_dir))
+ conn.sudo('sed -i "s|EDGE_INSTANCE_SIZE|{}|g" {}provisioning.yml'
+ .format(args.edge_instence_size, dlab_conf_dir))
+ conn.sudo('sed -i "s|SUBNET_ID|{}|g" {}provisioning.yml'
+ .format(args.subnet_id, dlab_conf_dir))
+ conn.sudo('sed -i "s|REGION|{}|g" {}provisioning.yml'
+ .format(args.region, dlab_conf_dir))
+ conn.sudo('sed -i "s|ZONE|{}|g" {}provisioning.yml'
+ .format(args.zone, dlab_conf_dir))
+ conn.sudo('sed -i "s|TAG_RESOURCE_ID|{}|g" {}provisioning.yml'
+ .format(args.tag_resource_id, dlab_conf_dir))
+ conn.sudo('sed -i "s|SG_IDS|{}|g" {}provisioning.yml'
+ .format(args.sg_ids, dlab_conf_dir))
+ conn.sudo('sed -i "s|SSN_INSTANCE_SIZE|{}|g" {}provisioning.yml'
+ .format(args.ssn_instance_size, dlab_conf_dir))
+ conn.sudo('sed -i "s|VPC2_ID|{}|g" {}provisioning.yml'
+ .format(args.vpc2_id, dlab_conf_dir))
+ conn.sudo('sed -i "s|SUBNET2_ID|{}|g" {}provisioning.yml'
+ .format(args.subnet2_id, dlab_conf_dir))
+ conn.sudo('sed -i "s|CONF_KEY_DIR|{}|g" {}provisioning.yml'
+ .format(args.conf_key_dir, dlab_conf_dir))
+ conn.sudo('sed -i "s|VPC_ID|{}|g" {}provisioning.yml'
+ .format(args.vpc_id, dlab_conf_dir))
+ conn.sudo('sed -i "s|PEERING_ID|{}|g" {}provisioning.yml'
+ .format(args.peering_id, dlab_conf_dir))
+ conn.sudo('sed -i "s|AZURE_RESOURCE_GROUP_NAME|{}|g" {}provisioning.yml'
+ .format(args.azure_resource_group_name, dlab_conf_dir))
+ conn.sudo('sed -i "s|AZURE_SSN_STORAGE_ACCOUNT_TAG|{}|g" {}provisioning.yml'
+ .format(args.azure_ssn_storage_account_tag, dlab_conf_dir))
+ conn.sudo('sed -i "s|AZURE_SHARED_STORAGE_ACCOUNT_TAG|{}|g" {}provisioning.yml'
+ .format(args.azure_shared_storage_account_tag, dlab_conf_dir))
+ conn.sudo('sed -i "s|AZURE_DATALAKE_TAG|{}|g" {}provisioning.yml'
+ .format(args.azure_datalake_tag, dlab_conf_dir))
+ conn.sudo('sed -i "s|AZURE_CLIENT_ID|{}|g" {}provisioning.yml'
+ .format(args.azure_client_id, dlab_conf_dir))
+ conn.sudo('sed -i "s|GCP_PROJECT_ID|{}|g" {}provisioning.yml'
+ .format(args.gcp_project_id, dlab_conf_dir))
+ conn.sudo('sed -i "s|LDAP_HOST|{}|g" {}provisioning.yml'
+ .format(args.ldap_host, dlab_conf_dir))
+ conn.sudo('sed -i "s|LDAP_DN|{}|g" {}provisioning.yml'
+ .format(args.ldap_dn, dlab_conf_dir))
+ conn.sudo('sed -i "s|LDAP_OU|{}|g" {}provisioning.yml'
+ .format(args.ldap_ou, dlab_conf_dir))
+ conn.sudo('sed -i "s|LDAP_USER_NAME|{}|g" {}provisioning.yml'
+ .format(args.ldap_user_name, dlab_conf_dir))
+ conn.sudo('sed -i "s|LDAP_USER_PASSWORD|{}|g" {}provisioning.yml'
+ .format(args.ldap_user_password, dlab_conf_dir))
conn.sudo('touch /home/{}/.ensure_dir/configure_supervisor_ensured'
.format(args.os_user))
except Exception as err:
@@ -252,12 +322,18 @@ def ensure_jar_endpoint():
web_path = '{}/webapp'.format(args.dlab_path)
if not exists(conn, web_path):
conn.run('mkdir -p {}'.format(web_path))
-
- conn.run('wget -P {} --user={} --password={} '
- 'https://{}/repository/packages/provisioning-service-'
- '2.1.jar --no-check-certificate'
- .format(web_path, args.repository_user,
- args.repository_pass, args.repository_address))
+ if args.cloud_provider == "aws":
+ conn.run('wget -P {} --user={} --password={} '
+ 'https://{}/repository/packages/aws/provisioning-service-'
+ '2.1.jar --no-check-certificate'
+ .format(web_path, args.repository_user,
+ args.repository_pass, args.repository_address))
+ elif args.cloud_provider == "gcp":
+ conn.run('wget -P {} --user={} --password={} '
+ 'https://{}/repository/packages/gcp/provisioning-service-'
+ '2.1.jar --no-check-certificate'
+ .format(web_path, args.repository_user,
+ args.repository_pass, args.repository_address))
conn.run('mv {0}/*.jar {0}/provisioning-service.jar'
.format(web_path))
conn.sudo('touch {}'.format(ensure_file))
@@ -276,6 +352,17 @@ def start_supervisor_endpoint():
sys.exit(1)
+def get_sources():
+ try:
+ conn.run("git clone https://github.com/apache/incubator-dlab.git {0}/sources".format(args.dlab_path))
+ if args.branch_name != "":
+ conn.run("cd {0}/sources && git checkout {1} && cd".format(args.dlab_path, args.branch_name))
+ except Exception as err:
+ logging.error('Failed to download sources: ', str(err))
+ traceback.print_exc()
+ sys.exit(1)
+
+
def pull_docker_images():
try:
ensure_file = ('/home/{}/.ensure_dir/docker_images_pulled'
@@ -286,77 +373,77 @@ def pull_docker_images():
args.repository_pass,
args.repository_address,
args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-base'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-edge'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-project'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-jupyter'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-rstudio'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-zeppelin'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-tensor'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-tensor-rstudio'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-deeplearning'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-dataengine-service'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker pull {}:{}/docker.dlab-dataengine'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-base docker.dlab-base'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-edge docker.dlab-edge'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-project docker.dlab-project'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-jupyter docker.dlab-jupyter'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-rstudio docker.dlab-rstudio'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-zeppelin '
+ conn.sudo('docker pull {}:{}/docker.dlab-base-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-edge-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-project-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-jupyter-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-rstudio-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-zeppelin-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-tensor-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-tensor-rstudio-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-deeplearning-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-dataengine-service-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker pull {}:{}/docker.dlab-dataengine-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-base-{} docker.dlab-base'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-edge-{} docker.dlab-edge'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-project-{} docker.dlab-project'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-jupyter-{} docker.dlab-jupyter'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-rstudio-{} docker.dlab-rstudio'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-zeppelin-{} '
'docker.dlab-zeppelin'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-tensor docker.dlab-tensor'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-tensor-rstudio '
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-tensor-{} docker.dlab-tensor'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-tensor-rstudio-{} '
'docker.dlab-tensor-rstudio'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-deeplearning '
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-deeplearning-{} '
'docker.dlab-deeplearning'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-dataengine-service '
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-dataengine-service-{} '
'docker.dlab-dataengine-service'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker tag {}:{}/docker.dlab-dataengine '
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker tag {}:{}/docker.dlab-dataengine-{} '
'docker.dlab-dataengine'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-base'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-edge'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-project'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-jupyter'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-rstudio'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-zeppelin'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-tensor'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-tensor-rstudio'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-deeplearning'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-dataengine-service'
- .format(args.repository_address, args.repository_port))
- conn.sudo('docker rmi {}:{}/docker.dlab-dataengine'
- .format(args.repository_address, args.repository_port))
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-base-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-edge-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-project-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-jupyter-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-rstudio-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-zeppelin-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-tensor-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-tensor-rstudio-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-deeplearning-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-dataengine-service-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
+ conn.sudo('docker rmi {}:{}/docker.dlab-dataengine-{}'
+ .format(args.repository_address, args.repository_port, args.cloud_provider))
conn.sudo('chown -R {0}:docker /home/{0}/.docker/'
.format(args.os_user))
conn.sudo('touch {}'.format(ensure_file))
@@ -369,15 +456,18 @@ def pull_docker_images():
def init_args():
global args
parser = argparse.ArgumentParser()
- parser.add_argument('--dlab_path', type=str, default='')
- parser.add_argument('--key_name', type=str, default='')
+ parser.add_argument('--dlab_path', type=str, default='/opt/dlab')
+ parser.add_argument('--key_name', type=str, default='', help='Name of admin key without .pem extension')
parser.add_argument('--endpoint_eip_address', type=str)
parser.add_argument('--pkey', type=str, default='')
parser.add_argument('--hostname', type=str, default='')
parser.add_argument('--os_user', type=str, default='dlab-user')
parser.add_argument('--cloud_provider', type=str, default='')
- parser.add_argument('--ssn_k8s_nlb_dns_name', type=str, default='')
- parser.add_argument('--ssn_k8s_alb_dns_name', type=str, default='')
+ parser.add_argument('--mongo_host', type=str, default='MONGO_HOST')
+ parser.add_argument('--mongo_port', type=str, default='27017')
+ parser.add_argument('--ss_host', type=str, default='')
+ parser.add_argument('--ss_port', type=str, default='8443')
+ parser.add_argument('--keycloack_host', type=str, default='')
# parser.add_argument('--mongo_password', type=str, default='')
parser.add_argument('--repository_address', type=str, default='')
parser.add_argument('--repository_port', type=str, default='')
@@ -388,6 +478,32 @@ def init_args():
parser.add_argument('--ssn_bucket_name', type=str, default='')
parser.add_argument('--endpoint_keystore_password', type=str, default='')
parser.add_argument('--keycloak_client_secret', type=str, default='')
+ parser.add_argument('--branch_name', type=str, default='DLAB-terraform') # change default
+ parser.add_argument('--conf_os', type=str, default='debian')
+ parser.add_argument('--service_base_name', type=str, default='')
+ parser.add_argument('--edge_instence_size', type=str, default='')
+ parser.add_argument('--subnet_id', type=str, default='')
+ parser.add_argument('--region', type=str, default='')
+ parser.add_argument('--zone', type=str, default='')
+ parser.add_argument('--tag_resource_id', type=str, default='')
+ parser.add_argument('--sg_ids', type=str, default='')
+ parser.add_argument('--ssn_instance_size', type=str, default='')
+ parser.add_argument('--vpc2_id', type=str, default='')
+ parser.add_argument('--subnet2_id', type=str, default='')
+ parser.add_argument('--conf_key_dir', type=str, default='/root/keys/', help='Should end by symbol /')
+ parser.add_argument('--vpc_id', type=str, default='')
+ parser.add_argument('--peering_id', type=str, default='')
+ parser.add_argument('--azure_resource_group_name', type=str, default='')
+ parser.add_argument('--azure_ssn_storage_account_tag', type=str, default='')
+ parser.add_argument('--azure_shared_storage_account_tag', type=str, default='')
+ parser.add_argument('--azure_datalake_tag', type=str, default='')
+ parser.add_argument('--azure_client_id', type=str, default='')
+ parser.add_argument('--gcp_project_id', type=str, default='')
+ parser.add_argument('--ldap_host', type=str, default='')
+ parser.add_argument('--ldap_dn', type=str, default='')
+ parser.add_argument('--ldap_ou', type=str, default='')
+ parser.add_argument('--ldap_user_name', type=str, default='')
+ parser.add_argument('--ldap_user_password', type=str, default='')
print(parser.parse_known_args())
args = parser.parse_known_args()[0]
@@ -474,6 +590,9 @@ def start_deploy():
logging.info("Ensure jar")
ensure_jar_endpoint()
+ logging.info("Downloading sources")
+ get_sources()
+
logging.info("Pulling docker images")
pull_docker_images()
diff --git a/infrastructure-provisioning/terraform/bin/deploy/provisioning.yml b/infrastructure-provisioning/terraform/bin/deploy/provisioning.yml
index 858b549..6eab11a 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/provisioning.yml
+++ b/infrastructure-provisioning/terraform/bin/deploy/provisioning.yml
@@ -36,23 +36,23 @@ devMode: ${DEV_MODE}
mongo:
- host: SSN_NLB
- port: 27017
+ host: MONGO_HOST
+ port: MONGO_PORT
username: admin
password: MONGO_PASSWORD
database: dlabdb
selfService:
protocol: https
- host: SSN_NLB
- port: 8443
+ host: SS_HOST
+ port: SS_PORT
jerseyClient:
timeout: 3s
connectionTimeout: 3s
securityService:
protocol: https
- host: SSN_NLB
+ host: DOESNT_MATTER
port: 8090
jerseyClient:
timeout: 20s
@@ -142,10 +142,38 @@ logging:
keycloakConfiguration:
realm: dlab
bearer-only: true
- auth-server-url: http://SSN_ALB/auth
+ auth-server-url: http://KEYCLOACK_HOST/auth
ssl-required: none
register-node-at-startup: true
register-node-period: 600
resource: dlab-ui
credentials:
secret: CLIENT_SECRET
+
+cloudProperties:
+ os: CONF_OS
+ serviceBaseName: SERVICE_BASE_NAME
+ edgeInstanceSize: EDGE_INSTANCE_SIZE
+ subnetId: SUBNET_ID
+ region: REGION
+ zone: ZONE
+ confTagResourceId: TAG_RESOURCE_ID
+ securityGroupIds: SG_IDS
+ ssnInstanceSize: SSN_INSTANCE_SIZE
+ notebookVpcId: VPC2_ID
+ notebookSubnetId: SUBNET2_ID
+ confKeyDir: CONF_KEY_DIR
+ vpcId: VPC_ID
+ peeringId: PEERING_ID
+ azureResourceGroupName: AZURE_RESOURCE_GROUP_NAME
+ ssnStorageAccountTagName: AZURE_SSN_STORAGE_ACCOUNT_TAG
+ sharedStorageAccountTagName: AZURE_SHARED_STORAGE_ACCOUNT_TAG
+ datalakeTagName: AZURE_DATALAKE_TAG
+ azureClientId: AZURE_CLIENT_ID
+ gcpProjectId: GCP_PROJECT_ID
+ ldap:
+ host: LDAP_HOST
+ dn: LDAP_DN
+ ou: LDAP_OU
+ user: LDAP_USER_NAME
+ password: LDAP_USER_PASSWORD
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/main/instance.tf b/infrastructure-provisioning/terraform/gcp/endpoint/main/instance.tf
index 52e0a5d..cfca293 100644
--- a/infrastructure-provisioning/terraform/gcp/endpoint/main/instance.tf
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/main/instance.tf
@@ -48,7 +48,7 @@ resource "google_compute_instance" "endpoint" {
}
service_account {
- email = google_service_account.endpoint_sa.email #"${var.project_name_var}-ssn-sa@${var.project_var}.iam.gserviceaccount.com"
+ email = google_service_account.endpoint_sa.email
scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute"]
}
@@ -63,4 +63,5 @@ resource "google_compute_instance" "endpoint" {
resource "google_compute_address" "static" {
name = local.endpoint_instance_ip
-}
\ No newline at end of file
+ count = var.static_ip == "" ? 1 : 0
+}
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/main/main.tf b/infrastructure-provisioning/terraform/gcp/endpoint/main/main.tf
index 76b4ada..3eab2a5 100644
--- a/infrastructure-provisioning/terraform/gcp/endpoint/main/main.tf
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/main/main.tf
@@ -21,7 +21,7 @@
provider "google" {
credentials = file(var.creds_file)
- project = var.project_name
+ project = var.project_id
region = var.region
zone = var.zone
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/main/variables.tf b/infrastructure-provisioning/terraform/gcp/endpoint/main/variables.tf
index ea68c9e..6c00f97 100644
--- a/infrastructure-provisioning/terraform/gcp/endpoint/main/variables.tf
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/main/variables.tf
@@ -19,7 +19,7 @@
#
# ******************************************************************************
-variable "project_name" {
+variable "project_id" {
default = ""
}
@@ -138,3 +138,7 @@ variable "path_to_pub_key" {
variable "product" {
default = "dlab"
}
+
+variable "static_ip" {
+ default = ""
+}
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
index 32f3b63..1fdbe1f 100644
--- a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.py
@@ -465,7 +465,7 @@ def init_args():
parser.add_argument('--os_user', type=str, default='dlab-user')
parser.add_argument('--cloud_provider', type=str, default='')
- parser.add_argument('--mongo_host', type=str, default='')
+ parser.add_argument('--mongo_host', type=str, default='MONGO_HOST')
parser.add_argument('--mongo_port', type=str, default='27017')
parser.add_argument('--ss_host', type=str, default='')
parser.add_argument('--ss_port', type=str, default='8443')
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml
index ce7f518..fd5fc9b 100644
--- a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml
@@ -104,25 +104,25 @@ server:
archivedLogFilenamePattern: ${LOG_ROOT_DIR}/provisioning/request-provisioning-%d{yyyy-MM-dd}.log.gz
archivedFileCount: 10
applicationConnectors:
- - type: http
-# - type: https
+# - type: http
+ - type: https
port: 8084
-# certAlias: dlab
-# validateCerts: true
-# keyStorePath: ${KEY_STORE_PATH}
-# keyStorePassword: ${KEY_STORE_PASSWORD}
-# trustStorePath: ${TRUST_STORE_PATH}
-# trustStorePassword: ${TRUST_STORE_PASSWORD}
+ certAlias: dlab
+ validateCerts: true
+ keyStorePath: ${KEY_STORE_PATH}
+ keyStorePassword: ${KEY_STORE_PASSWORD}
+ trustStorePath: ${TRUST_STORE_PATH}
+ trustStorePassword: ${TRUST_STORE_PASSWORD}
adminConnectors:
- - type: http
-# - type: https
+# - type: http
+ - type: https
port: 8085
-# certAlias: dlab
-# validateCerts: true
-# keyStorePath: ${KEY_STORE_PATH}
-# keyStorePassword: ${KEY_STORE_PASSWORD}
-# trustStorePath: ${TRUST_STORE_PATH}
-# trustStorePassword: ${TRUST_STORE_PASSWORD}
+ certAlias: dlab
+ validateCerts: true
+ keyStorePath: ${KEY_STORE_PATH}
+ keyStorePassword: ${KEY_STORE_PASSWORD}
+ trustStorePath: ${TRUST_STORE_PATH}
+ trustStorePassword: ${TRUST_STORE_PASSWORD}
logging:
level: INFO
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org