You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by bh...@apache.org on 2019/09/18 13:45:53 UTC
[incubator-dlab] branch DLAB-terraform updated: DLAB-000 moved
cloud settings from self-service to provisioning
This is an automated email from the ASF dual-hosted git repository.
bhliva pushed a commit to branch DLAB-terraform
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/DLAB-terraform by this push:
new 0d98c6e DLAB-000 moved cloud settings from self-service to provisioning
0d98c6e is described below
commit 0d98c6e326483c0f87830ed7c8e301b56b8729dd
Author: bhliva <bo...@epam.com>
AuthorDate: Wed Sep 18 16:45:19 2019 +0300
DLAB-000 moved cloud settings from self-service to provisioning
---
services/provisioning-service/provisioning.yml | 30 +++++++-
.../epam/dlab/backendapi/CloudConfiguration.java | 40 ++++++++++
...rovisioningServiceApplicationConfiguration.java | 7 ++
.../backendapi/core/commands/CommandBuilder.java | 86 ++++++++++++++++++++++
.../dlab/backendapi/core/CommandBuilderTest.java | 65 ----------------
services/self-service/self-service.yml | 29 --------
.../conf/SelfServiceApplicationConfiguration.java | 7 --
.../dropwizard/listeners/MongoStartupListener.java | 47 ------------
8 files changed, 162 insertions(+), 149 deletions(-)
diff --git a/services/provisioning-service/provisioning.yml b/services/provisioning-service/provisioning.yml
index 1fbf2cf..bd39c90 100644
--- a/services/provisioning-service/provisioning.yml
+++ b/services/provisioning-service/provisioning.yml
@@ -96,4 +96,32 @@ keycloakConfiguration:
register-node-period: 600
resource: KEYCLOAK_CLIENT_NAME
credentials:
- secret: KEYCLOAK_CLIENT_SECRET
\ No newline at end of file
+ secret: KEYCLOAK_CLIENT_SECRET
+
+cloudProperties:
+ os: CONF_OS
+ serviceBaseName: SERVICE_BASE_NAME
+ edgeInstanceSize: EDGE_INSTANCE_SIZE
+ subnetId: SUBNET_ID
+ region: REGION
+ zone: ZONE
+ confTagResourceId: TAG_RESOURCE_ID
+ securityGroupIds: SG_IDS
+ ssnInstanceSize: SSN_INSTANCE_SIZE
+ notebookVpcId: VPC2_ID
+ notebookSubnetId: SUBNET2_ID
+ confKeyDir: CONF_KEY_DIR
+ vpcId: VPC_ID
+ peeringId: PEERING_ID
+ azureResourceGroupName: AZURE_RESOURCE_GROUP_NAME
+ ssnStorageAccountTagName: AZURE_SSN_STORAGE_ACCOUNT_TAG
+ sharedStorageAccountTagName: AZURE_SHARED_STORAGE_ACCOUNT_TAG
+ datalakeTagName: AZURE_DATALAKE_TAG
+ azureClientId: AZURE_CLIENT_ID
+ gcpProjectId: GCP_PROJECT_ID
+ ldap:
+ host: LDAP_HOST
+ dn: LDAP_DN
+ ou: LDAP_OU
+ user: LDAP_USER_NAME
+ password: LDAP_USER_PASSWORD
\ No newline at end of file
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/CloudConfiguration.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/CloudConfiguration.java
new file mode 100644
index 0000000..3f9339a
--- /dev/null
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/CloudConfiguration.java
@@ -0,0 +1,40 @@
+package com.epam.dlab.backendapi;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+@Data
+public class CloudConfiguration {
+
+ private final String os;
+ private final String serviceBaseName;
+ private final String edgeInstanceSize;
+ private final String subnetId;
+ private final String region;
+ private final String zone;
+ private final String confTagResourceId;
+ private final String securityGroupIds;
+ private final String ssnInstanceSize;
+ private final String notebookVpcId;
+ private final String notebookSubnetId;
+ private final String confKeyDir;
+ private final String vpcId;
+ private final String azureResourceGroupName;
+ private final String ssnStorageAccountTagName;
+ private final String sharedStorageAccountTagName;
+ private final String datalakeTagName;
+ private final String azureClientId;
+ private final String peeringId;
+ private final String gcpProjectId;
+ @JsonProperty("ldap")
+ private final LdapConfig ldapConfig;
+
+ @Data
+ public static class LdapConfig {
+ private final String host;
+ private final String dn;
+ private final String ou;
+ private final String user;
+ private final String password;
+ }
+}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/ProvisioningServiceApplicationConfiguration.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/ProvisioningServiceApplicationConfiguration.java
index adf5198..912167a 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/ProvisioningServiceApplicationConfiguration.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/ProvisioningServiceApplicationConfiguration.java
@@ -96,6 +96,9 @@ public class ProvisioningServiceApplicationConfiguration extends ServiceConfigur
private KeycloakConfiguration keycloakConfiguration = new KeycloakConfiguration();
+ @JsonProperty("cloudProperties")
+ private CloudConfiguration cloudConfiguration;
+
public boolean isHandlersPersistenceEnabled() {
return handlersPersistenceEnabled;
}
@@ -186,4 +189,8 @@ public class ProvisioningServiceApplicationConfiguration extends ServiceConfigur
public KeycloakConfiguration getKeycloakConfiguration() {
return keycloakConfiguration;
}
+
+ public CloudConfiguration getCloudConfiguration() {
+ return cloudConfiguration;
+ }
}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/commands/CommandBuilder.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/commands/CommandBuilder.java
index 7721fe8..fa61bdd 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/commands/CommandBuilder.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/commands/CommandBuilder.java
@@ -19,10 +19,18 @@
package com.epam.dlab.backendapi.core.commands;
+import com.epam.dlab.backendapi.CloudConfiguration;
+import com.epam.dlab.backendapi.ProvisioningServiceApplicationConfiguration;
+import com.epam.dlab.cloud.CloudProvider;
import com.epam.dlab.dto.ResourceBaseDTO;
+import com.epam.dlab.dto.aws.AwsCloudSettings;
+import com.epam.dlab.dto.azure.AzureCloudSettings;
+import com.epam.dlab.dto.base.CloudSettings;
+import com.epam.dlab.dto.gcp.GcpCloudSettings;
import com.epam.dlab.util.JsonGenerator;
import com.epam.dlab.util.SecurityUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.inject.Inject;
import com.google.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
@@ -30,11 +38,19 @@ import lombok.extern.slf4j.Slf4j;
@Singleton
public class CommandBuilder {
+ private final ProvisioningServiceApplicationConfiguration conf;
+
+ @Inject
+ public CommandBuilder(ProvisioningServiceApplicationConfiguration conf) {
+ this.conf = conf;
+ }
+
public String buildCommand(RunDockerCommand runDockerCommand, ResourceBaseDTO<?> resourceBaseDTO) throws JsonProcessingException {
StringBuilder builder = new StringBuilder();
if (resourceBaseDTO != null) {
builder.append("echo -e '");
try {
+ resourceBaseDTO.setCloudSettings(getCloudSettings(resourceBaseDTO.getCloudSettings()));
String str = JsonGenerator.generateJson(resourceBaseDTO);
log.info("Serialized DTO to: {}", SecurityUtils.hideCreds(str));
builder.append(str);
@@ -48,4 +64,74 @@ public class CommandBuilder {
builder.append(runDockerCommand.toCMD());
return builder.toString();
}
+
+ private CloudSettings getCloudSettings(CloudSettings settings) {
+ final CloudProvider cloudProvider = conf.getCloudProvider();
+ final CloudConfiguration cloudConfiguration = conf.getCloudConfiguration();
+ final CloudConfiguration.LdapConfig ldapConfig = cloudConfiguration.getLdapConfig();
+ if (cloudProvider == CloudProvider.AWS) {
+ return awsCloudSettings(settings, cloudConfiguration, ldapConfig);
+ } else if (cloudProvider == CloudProvider.GCP) {
+ return gcpCloudSettings(settings, cloudConfiguration, ldapConfig);
+ } else if (cloudProvider == CloudProvider.AZURE) {
+ return azureCloudSettings(settings, cloudConfiguration);
+ } else {
+ throw new UnsupportedOperationException("Unsupported cloud provider " + cloudProvider.getName());
+ }
+ }
+
+ private AzureCloudSettings azureCloudSettings(CloudSettings settings, CloudConfiguration cloudConfiguration) {
+ return AzureCloudSettings.builder()
+ .azureRegion(cloudConfiguration.getRegion())
+ .azureResourceGroupName(cloudConfiguration.getAzureResourceGroupName())
+ .azureSecurityGroupName(cloudConfiguration.getSecurityGroupIds())
+ .azureSubnetName(cloudConfiguration.getSubnetId())
+ .azureVpcName(cloudConfiguration.getVpcId())
+ .confKeyDir(cloudConfiguration.getConfKeyDir())
+ .azureIamUser(settings.getIamUser()).build();
+ }
+
+ private GcpCloudSettings gcpCloudSettings(CloudSettings settings, CloudConfiguration cloudConfiguration,
+ CloudConfiguration.LdapConfig ldapConfig) {
+ return GcpCloudSettings.builder()
+ .projectId(cloudConfiguration.getGcpProjectId())
+ .vpcName(cloudConfiguration.getVpcId())
+ .subnetName(cloudConfiguration.getSubnetId())
+ .zone(cloudConfiguration.getZone())
+ .region(cloudConfiguration.getRegion())
+ .ldapDn(ldapConfig.getDn())
+ .ldapHost(ldapConfig.getHost())
+ .ldapOu(ldapConfig.getOu())
+ .ldapUser(ldapConfig.getUser())
+ .ldapPassword(ldapConfig.getPassword())
+ .sbn(cloudConfiguration.getServiceBaseName())
+ .cloud(conf.getCloudProvider().getName())
+ .os(cloudConfiguration.getOs())
+ .confKeyDir(cloudConfiguration.getConfKeyDir())
+ .gcpIamUser(settings.getIamUser()).build();
+ }
+
+ private AwsCloudSettings awsCloudSettings(CloudSettings settings, CloudConfiguration cloudConfiguration,
+ CloudConfiguration.LdapConfig ldapConfig) {
+ return AwsCloudSettings.builder()
+ .awsRegion(cloudConfiguration.getRegion())
+ .awsSecurityGroupIds(cloudConfiguration.getSecurityGroupIds())
+ .awsSubnetId(cloudConfiguration.getSubnetId())
+ .awsVpcId(cloudConfiguration.getVpcId())
+ .confTagResourceId(cloudConfiguration.getConfTagResourceId())
+ .awsNotebookSubnetId(cloudConfiguration.getNotebookSubnetId())
+ .awsNotebookVpcId(cloudConfiguration.getNotebookVpcId())
+ .awsIamUser(settings.getIamUser())
+ .zone(cloudConfiguration.getZone())
+ .ldapDn(ldapConfig.getDn())
+ .ldapHost(ldapConfig.getHost())
+ .ldapOu(ldapConfig.getOu())
+ .ldapUser(ldapConfig.getUser())
+ .ldapPassword(ldapConfig.getPassword())
+ .sbn(cloudConfiguration.getServiceBaseName())
+ .cloud(conf.getCloudProvider().getName())
+ .os(cloudConfiguration.getOs())
+ .confKeyDir(cloudConfiguration.getConfKeyDir())
+ .build();
+ }
}
diff --git a/services/provisioning-service/src/test/java/com/epam/dlab/backendapi/core/CommandBuilderTest.java b/services/provisioning-service/src/test/java/com/epam/dlab/backendapi/core/CommandBuilderTest.java
deleted file mode 100644
index fbbeeaf..0000000
--- a/services/provisioning-service/src/test/java/com/epam/dlab/backendapi/core/CommandBuilderTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.backendapi.core;
-
-import com.epam.dlab.backendapi.core.commands.CommandBuilder;
-import com.epam.dlab.backendapi.core.commands.DockerCommands;
-import com.epam.dlab.backendapi.core.commands.RunDockerCommand;
-import com.epam.dlab.dto.aws.computational.ComputationalCreateAws;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class CommandBuilderTest {
-
- String rootKeysVolume = "rkv";
- String responseVolume = "rv";
- String requestID = "rID";
- String toDescribe = "ubuntu";
-
- @Test
- public void testBuildCommand() throws JsonProcessingException {
- RunDockerCommand dockerBaseCommand = new RunDockerCommand()
- .withInteractive()
- .withAtach("STDIN")
- .withVolumeForRootKeys(rootKeysVolume)
- .withVolumeForResponse(responseVolume)
- .withRequestId(requestID)
- .withActionDescribe(toDescribe);
-
-
- ComputationalCreateAws computationalCreateAws = new ComputationalCreateAws().withServiceBaseName("someName");
- CommandBuilder commandBuilder = new CommandBuilder();
- String command = commandBuilder.buildCommand(dockerBaseCommand, computationalCreateAws);
- System.out.println(command);
-
- assertEquals("echo -e '{\"@class\":\"com.epam.dlab.dto.aws.computational.ComputationalCreateAws\"," +
- "\"conf_service_base_name\":\"someName\"}' | docker run -i -a STDIN -v rkv:/root/keys -v " +
- "rv:/response -e \"request_id=rID\" ubuntu --action describe",
- command);
- }
-
- @Test
- public void extractUUIDSuccess() {
- String uuid = DockerCommands.extractUUID("edge_user_name_2fa2fec8-4d30-4563-b78a-ab1f7539c862.json");
- assertEquals("2fa2fec8-4d30-4563-b78a-ab1f7539c862", uuid);
- }
-}
diff --git a/services/self-service/self-service.yml b/services/self-service/self-service.yml
index 7aa0080..4f67acf 100644
--- a/services/self-service/self-service.yml
+++ b/services/self-service/self-service.yml
@@ -193,32 +193,3 @@ jerseyClient:
gzipEnabledForRequests: false
chunkedEncodingEnabled: true
-cloudProperties:
- os: CONF_OS
- serviceBaseName: SERVICE_BASE_NAME
- edgeInstanceSize: EDGE_INSTANCE_SIZE
- subnetId: SUBNET_ID
- region: REGION
- zone: ZONE
- confTagResourceId: TAG_RESOURCE_ID
- securityGroupIds: SG_IDS
- ssnInstanceSize: SSN_INSTANCE_SIZE
- notebookVpcId: VPC2_ID
- notebookSubnetId: SUBNET2_ID
- confKeyDir: CONF_KEY_DIR
- vpcId: VPC_ID
- peeringId: PEERING_ID
- azureResourceGroupName: AZURE_RESOURCE_GROUP_NAME
- ssnStorageAccountTagName: AZURE_SSN_STORAGE_ACCOUNT_TAG
- sharedStorageAccountTagName: AZURE_SHARED_STORAGE_ACCOUNT_TAG
- datalakeTagName: AZURE_DATALAKE_TAG
- azureClientId: AZURE_CLIENT_ID
- gcpProjectId: GCP_PROJECT_ID
- ldap:
- host: LDAP_HOST
- dn: LDAP_DN
- ou: LDAP_OU
- user: LDAP_USER_NAME
- password: LDAP_USER_PASSWORD
-
-
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/conf/SelfServiceApplicationConfiguration.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/conf/SelfServiceApplicationConfiguration.java
index 9d070c8..f2898e6 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/conf/SelfServiceApplicationConfiguration.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/conf/SelfServiceApplicationConfiguration.java
@@ -129,9 +129,6 @@ public class SelfServiceApplicationConfiguration extends ServiceConfiguration {
@NotNull
private Map<String, String> guacamole;
- @JsonProperty("cloudProperties")
- private CloudConfiguration cloudConfiguration;
-
private KeycloakConfiguration keycloakConfiguration = new KeycloakConfiguration();
public Map<String, String> getGuacamole() {
@@ -261,10 +258,6 @@ public class SelfServiceApplicationConfiguration extends ServiceConfiguration {
return mongoMigrationEnabled;
}
- public CloudConfiguration getCloudConfiguration() {
- return cloudConfiguration;
- }
-
@NotNull
public RESTServiceFactory getMavenApiFactory() {
return mavenApiFactory;
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dropwizard/listeners/MongoStartupListener.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dropwizard/listeners/MongoStartupListener.java
index 01ef1e5..f3075eb 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dropwizard/listeners/MongoStartupListener.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dropwizard/listeners/MongoStartupListener.java
@@ -1,11 +1,9 @@
package com.epam.dlab.backendapi.dropwizard.listeners;
-import com.epam.dlab.backendapi.conf.CloudConfiguration;
import com.epam.dlab.backendapi.conf.SelfServiceApplicationConfiguration;
import com.epam.dlab.backendapi.dao.SettingsDAO;
import com.epam.dlab.backendapi.dao.UserRoleDao;
import com.epam.dlab.backendapi.resources.dto.UserRoleDto;
-import com.epam.dlab.cloud.CloudProvider;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Inject;
@@ -40,51 +38,6 @@ public class MongoStartupListener implements ServerLifecycleListener {
@Override
public void serverStarted(Server server) {
insertRoles();
- insertCloudSettings();
- }
-
- private void insertCloudSettings() {
- log.debug("Populating DLab cloud properties into database");
- final CloudConfiguration cloudConfiguration = configuration.getCloudConfiguration();
- settingsDAO.setServiceBaseName(cloudConfiguration.getServiceBaseName());
- settingsDAO.setConfKeyDir(cloudConfiguration.getConfKeyDir());
- settingsDAO.setConfOsFamily(cloudConfiguration.getOs());
- settingsDAO.setConfTagResourceId(cloudConfiguration.getConfTagResourceId());
- final CloudConfiguration.LdapConfig ldapConfig = cloudConfiguration.getLdapConfig();
- settingsDAO.setLdapDn(ldapConfig.getDn());
- settingsDAO.setLdapHost(ldapConfig.getHost());
- settingsDAO.setLdapOu(ldapConfig.getOu());
- settingsDAO.setLdapUser(ldapConfig.getUser());
- settingsDAO.setLdapPassword(ldapConfig.getPassword());
- settingsDAO.setSsnStorageAccountTagName(cloudConfiguration.getSsnStorageAccountTagName());
- settingsDAO.setPeeringId(cloudConfiguration.getPeeringId());
-
- final CloudProvider cloudProvider = configuration.getCloudProvider();
- if (cloudProvider == CloudProvider.AWS) {
- settingsDAO.setAwsZone(cloudConfiguration.getZone());
- settingsDAO.setAwsRegion(cloudConfiguration.getRegion());
- settingsDAO.setAwsVpcId(cloudConfiguration.getVpcId());
- settingsDAO.setAwsSubnetId(cloudConfiguration.getSubnetId());
- settingsDAO.setAwsNotebookVpcId(cloudConfiguration.getNotebookVpcId());
- settingsDAO.setAwsNotebookSubnetId(cloudConfiguration.getNotebookSubnetId());
- settingsDAO.setAwsSecurityGroups(cloudConfiguration.getSecurityGroupIds());
- } else if (cloudProvider == CloudProvider.AZURE) {
- settingsDAO.setAzureRegion(cloudConfiguration.getRegion());
- settingsDAO.setAzureVpcName(cloudConfiguration.getVpcId());
- settingsDAO.setAzureSubnetName(cloudConfiguration.getSubnetId());
- settingsDAO.setAzureDataLakeClientId(cloudConfiguration.getAzureClientId());
- settingsDAO.setAzureResourceGroupName(cloudConfiguration.getAzureResourceGroupName());
- settingsDAO.setAzureSecurityGroupName(cloudConfiguration.getSecurityGroupIds());
- settingsDAO.setAzureDataLakeNameTag(cloudConfiguration.getDatalakeTagName());
- settingsDAO.setSsnStorageAccountTagName(cloudConfiguration.getSsnStorageAccountTagName());
- settingsDAO.setSharedStorageAccountTagName(cloudConfiguration.getSharedStorageAccountTagName());
- } else if (cloudProvider == CloudProvider.GCP) {
- settingsDAO.setGcpProjectId(cloudConfiguration.getGcpProjectId());
- settingsDAO.setGcpRegion(cloudConfiguration.getRegion());
- settingsDAO.setGcpZone(cloudConfiguration.getZone());
- settingsDAO.setGcpSubnetName(cloudConfiguration.getSubnetId());
- settingsDAO.setGcpVpcName(cloudConfiguration.getVpcId());
- }
}
private void insertRoles() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org