You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datalab.apache.org by yk...@apache.org on 2021/02/15 11:56:24 UTC
[incubator-datalab] 03/04: [DATALAB-2228] - added possibility of
change properties on external endpoint
This is an automated email from the ASF dual-hosted git repository.
ykinash pushed a commit to branch DATALAB-2228
in repository https://gitbox.apache.org/repos/asf/incubator-datalab.git
commit 6748a709a90763ef044579c33396e93d5de07cb5
Author: KinashYurii <ur...@gmail.com>
AuthorDate: Fri Feb 12 14:06:08 2021 +0200
[DATALAB-2228] - added possibility of change properties on external endpoint
---
.../resources/admin/ChangePropertiesResource.java | 119 +++++++++++++++++++--
.../service/impl/DynamicChangeProperties.java | 94 +++++++++++-----
2 files changed, 175 insertions(+), 38 deletions(-)
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/admin/ChangePropertiesResource.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/admin/ChangePropertiesResource.java
index 2386b86..1c51385 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/admin/ChangePropertiesResource.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/admin/ChangePropertiesResource.java
@@ -10,19 +10,33 @@ import lombok.NoArgsConstructor;
import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import java.util.List;
+import java.util.Map;
-@Path("admin")
+@Path("configuration")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@NoArgsConstructor
public class ChangePropertiesResource {
+ private static final String SELF_SERVICE = "self-service.yml";
+ //services/self-service/self-service.yml";
+ private static final String SELF_SERVICE_PROP_PATH = "/opt/datalab/conf/self-service.yml";
+ private static final String PROVISIONING_SERVICE = "provisioning.yml";
+ //"services/provisioning-service/provisioning.yml";
+ private static final String PROVISIONING_SERVICE_PROP_PATH = "/opt/datalab/conf/provisioning.yml";
+ private static final String BILLING_SERVICE = "billing.yml";
+ //"services/billing-aws/billing.yml";
+ //"services/billing-azure/billing.yml";
+ //"services/billing-gcp/billing.yml";
+ private static final String BILLING_SERVICE_PROP_PATH = "/opt/datalab/conf/billing.yml";
+
@GET
@Path("/self-service")
public Response getSelfServiceProperties(@Auth UserInfo userInfo) {
if (UserRoles.isAdmin(userInfo)) {
return Response
- .ok(DynamicChangeProperties.getSelfServiceProperties())
+ .ok(DynamicChangeProperties.getProperties(SELF_SERVICE_PROP_PATH, SELF_SERVICE))
.build();
} else {
return Response
@@ -36,7 +50,7 @@ public class ChangePropertiesResource {
public Response getProvisioningServiceProperties(@Auth UserInfo userInfo) {
if (UserRoles.isAdmin(userInfo)) {
return Response
- .ok(DynamicChangeProperties.getProvisioningServiceProperties())
+ .ok(DynamicChangeProperties.getProperties(PROVISIONING_SERVICE_PROP_PATH, PROVISIONING_SERVICE))
.build();
} else {
return Response
@@ -50,7 +64,7 @@ public class ChangePropertiesResource {
public Response getBillingServiceProperties(@Auth UserInfo userInfo) {
if (UserRoles.isAdmin(userInfo)) {
return Response
- .ok(DynamicChangeProperties.getBillingServiceProperties())
+ .ok(DynamicChangeProperties.getProperties(BILLING_SERVICE_PROP_PATH, BILLING_SERVICE))
.build();
} else {
return Response
@@ -63,7 +77,7 @@ public class ChangePropertiesResource {
@Path("/self-service")
public Response overwriteSelfServiceProperties(@Auth UserInfo userInfo, YmlDTO ymlDTO) {
if (UserRoles.isAdmin(userInfo)) {
- DynamicChangeProperties.overwriteSelfServiceProperties(ymlDTO.getYmlString());
+ DynamicChangeProperties.overwriteProperties(SELF_SERVICE_PROP_PATH, SELF_SERVICE, ymlDTO.getYmlString());
return Response.ok().build();
} else {
return Response
@@ -76,7 +90,8 @@ public class ChangePropertiesResource {
@Path("/provisioning-service")
public Response overwriteProvisioningServiceProperties(@Auth UserInfo userInfo, YmlDTO ymlDTO) {
if (UserRoles.isAdmin(userInfo)) {
- DynamicChangeProperties.overwriteProvisioningServiceProperties(ymlDTO.getYmlString());
+ DynamicChangeProperties.overwriteProperties(PROVISIONING_SERVICE_PROP_PATH, PROVISIONING_SERVICE,
+ ymlDTO.getYmlString());
return Response.ok().build();
} else {
return Response
@@ -89,7 +104,90 @@ public class ChangePropertiesResource {
@Path("/billing")
public Response overwriteBillingServiceProperties(@Auth UserInfo userInfo, YmlDTO ymlDTO) {
if (UserRoles.isAdmin(userInfo)) {
- DynamicChangeProperties.overwriteBillingServiceProperties(ymlDTO.getYmlString());
+ DynamicChangeProperties.overwriteProperties(BILLING_SERVICE_PROP_PATH, BILLING_SERVICE, ymlDTO.getYmlString());
+ return Response.ok().build();
+ } else {
+ return Response
+ .status(Response.Status.FORBIDDEN)
+ .build();
+ }
+ }
+
+ @GET
+ @Path("multiple/self-service")
+ public Response getAllSelfServiceProperties(@Auth UserInfo userInfo) {
+ if (UserRoles.isAdmin(userInfo)) {
+ return Response
+ .ok(DynamicChangeProperties.getPropertiesWithExternal(SELF_SERVICE_PROP_PATH, SELF_SERVICE, userInfo))
+ .build();
+ } else {
+ return Response
+ .status(Response.Status.FORBIDDEN)
+ .build();
+ }
+ }
+
+ @GET
+ @Path("multiple/provisioning-service")
+ public Response getAllProvisioningServiceProperties(@Auth UserInfo userInfo) {
+ if (UserRoles.isAdmin(userInfo)) {
+ return Response
+ .ok(DynamicChangeProperties.getPropertiesWithExternal(PROVISIONING_SERVICE_PROP_PATH, PROVISIONING_SERVICE, userInfo))
+ .build();
+ } else {
+ return Response
+ .status(Response.Status.FORBIDDEN)
+ .build();
+ }
+ }
+
+ @GET
+ @Path("multiple/billing")
+ public Response getAllBillingServiceProperties(@Auth UserInfo userInfo) {
+ if (UserRoles.isAdmin(userInfo)) {
+ return Response
+ .ok(DynamicChangeProperties.getPropertiesWithExternal(BILLING_SERVICE_PROP_PATH, BILLING_SERVICE, userInfo))
+ .build();
+ } else {
+ return Response
+ .status(Response.Status.FORBIDDEN)
+ .build();
+ }
+ }
+
+ @POST
+ @Path("multiple/self-service")
+ public Response overwriteAllSelfServiceProperties(@Auth UserInfo userInfo, Map<String, YmlDTO> ymlDTO) {
+ if (UserRoles.isAdmin(userInfo)) {
+ DynamicChangeProperties.overwritePropertiesWithExternal(SELF_SERVICE_PROP_PATH, SELF_SERVICE, ymlDTO, userInfo);
+ return Response.ok().build();
+ } else {
+ return Response
+ .status(Response.Status.FORBIDDEN)
+ .build();
+ }
+ }
+
+ @POST
+ @Path("multiple/provisioning-service")
+ public Response overwriteAllProvisioningServiceProperties(@Auth UserInfo userInfo, Map<String, YmlDTO> ymlDTO) {
+ if (UserRoles.isAdmin(userInfo)) {
+ DynamicChangeProperties.overwritePropertiesWithExternal(PROVISIONING_SERVICE_PROP_PATH, PROVISIONING_SERVICE,
+ ymlDTO, userInfo);
+ return Response.ok().build();
+ } else {
+ return Response
+ .status(Response.Status.FORBIDDEN)
+ .build();
+ }
+ }
+
+ @POST
+ @Path("multiple/billing")
+ public Response overwriteAllBillingServiceProperties(@Auth UserInfo userInfo, Map<String, YmlDTO> ymlDTO) {
+ if (UserRoles.isAdmin(userInfo)) {
+ DynamicChangeProperties.overwritePropertiesWithExternal(BILLING_SERVICE_PROP_PATH, BILLING_SERVICE, ymlDTO,
+ userInfo);
return Response.ok().build();
} else {
return Response
@@ -103,10 +201,11 @@ public class ChangePropertiesResource {
public Response restart(@Auth UserInfo userInfo,
@QueryParam("billing") boolean billing,
@QueryParam("provserv") boolean provserv,
- @QueryParam("ui") boolean ui) {
+ @QueryParam("ui") boolean ui,
+ @QueryParam("endpoints") List<String> endpoints) {
if (UserRoles.isAdmin(userInfo)) {
- DynamicChangeProperties.restart(billing, provserv, ui);
- return Response.ok().build();
+ DynamicChangeProperties.restart(billing, provserv, ui);
+ return Response.ok().build();
} else {
return Response
.status(Response.Status.FORBIDDEN)
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/DynamicChangeProperties.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/DynamicChangeProperties.java
index 5ea27f8..b028802 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/DynamicChangeProperties.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/DynamicChangeProperties.java
@@ -1,10 +1,16 @@
package com.epam.datalab.backendapi.service.impl;
+import com.epam.datalab.auth.UserInfo;
import com.epam.datalab.backendapi.annotation.Audit;
+import com.epam.datalab.backendapi.dao.EndpointDAO;
+import com.epam.datalab.backendapi.domain.EndpointDTO;
+import com.epam.datalab.backendapi.resources.dto.YmlDTO;
import com.epam.datalab.exceptions.DynamicChangePropertiesException;
+import com.epam.datalab.rest.client.RESTService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
+import javax.inject.Inject;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
@@ -16,6 +22,7 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import static com.epam.datalab.backendapi.domain.AuditActionEnum.RECONFIGURE;
import static com.epam.datalab.backendapi.domain.AuditResourceTypeEnum.EDGE_NODE;
@@ -23,19 +30,9 @@ import static com.epam.datalab.backendapi.domain.AuditResourceTypeEnum.EDGE_NODE
@Slf4j
public class DynamicChangeProperties {
- private static final String SELF_SERVICE = "self-service.yml";
- //services/self-service/self-service.yml";
- private static final String SELF_SERVICE_PROP_PATH = "/opt/datalab/conf/self-service.yml";
+
private static final String SELF_SERVICE_SUPERVISORCTL_RUN_NAME = " ui ";
- private static final String PROVISIONING_SERVICE = "provisioning.yml";
- //"services/provisioning-service/provisioning.yml";
- private static final String PROVISIONING_SERVICE_PROP_PATH = "/opt/datalab/conf/provisioning.yml";
private static final String PROVISIONING_SERVICE_SUPERVISORCTL_RUN_NAME = " provserv ";
- private static final String BILLING_SERVICE = "billing.yml";
- //"services/billing-aws/billing.yml";
- //"services/billing-azure/billing.yml";
- //"services/billing-gcp/billing.yml";
- private static final String BILLING_SERVICE_PROP_PATH = "/opt/datalab/conf/billing.yml";
private static final String BILLING_SERVICE_SUPERVISORCTL_RUN_NAME = " billing ";
private static final String SECRET_REGEX = "((.*)[sS]ecret(.*)|password): (.*)";
private static final String SECRET_REPLACEMENT_FORMAT = " ***********";
@@ -65,39 +62,73 @@ public class DynamicChangeProperties {
"# specific language governing permissions and limitations\n" +
"# under the License.\n" +
"#\n" +
- "# ******************************************************************************\n";
+ "# ******************************************************************************";
private static final int DEFAULT_VALUE_PLACE = 1;
private static final int DEFAULT_NAME_PLACE = 0;
- @Audit(action = RECONFIGURE, type = EDGE_NODE)
- public static String getSelfServiceProperties() {
- return readFileAsString(SELF_SERVICE_PROP_PATH, SELF_SERVICE);
- }
+ @Inject
+ private static RESTService externalSelfService;
+ @Inject
+ private static EndpointDAO endpointDAO;
@Audit(action = RECONFIGURE, type = EDGE_NODE)
- public static String getProvisioningServiceProperties() {
- return readFileAsString(PROVISIONING_SERVICE_PROP_PATH, PROVISIONING_SERVICE);
+ public static String getProperties(String path, String name) {
+ return readFileAsString(path, name);
}
@Audit(action = RECONFIGURE, type = EDGE_NODE)
- public static String getBillingServiceProperties() {
- return readFileAsString(BILLING_SERVICE_PROP_PATH, BILLING_SERVICE);
+ public static void overwriteProperties(String path, String name, String ymlString) {
+ writeFileFromString(ymlString, name, path);
}
@Audit(action = RECONFIGURE, type = EDGE_NODE)
- public static void overwriteSelfServiceProperties(String ymlString) {
- writeFileFromString(ymlString, SELF_SERVICE, SELF_SERVICE_PROP_PATH);
+ public static Map<String, String> getPropertiesWithExternal(String path, String name, UserInfo userInfo) {
+ List<EndpointDTO> externalEndpoints = endpointDAO.getEndpointsWithStatus("ACTIVE");
+ Map<String, String> endpoints = externalEndpoints.stream()
+ .filter(endpointDTO -> !endpointDTO.getName().equals("Local"))
+ .collect(Collectors.toMap(EndpointDTO::getName,
+ dto -> readFileAsString(path, name, dto, userInfo)));
+ endpoints.put("Local", getProperties(path, name));
+ return endpoints;
+
}
@Audit(action = RECONFIGURE, type = EDGE_NODE)
- public static void overwriteProvisioningServiceProperties(String ymlString) {
- writeFileFromString(ymlString, PROVISIONING_SERVICE, PROVISIONING_SERVICE_PROP_PATH);
+ public static void overwritePropertiesWithExternal(String path, String name, Map<String, YmlDTO> ymlDTOS,
+ UserInfo userInfo) {
+
+ List<EndpointDTO> allEndpoints = endpointDAO.getEndpointsWithStatus("ACTIVE");
+ Map<EndpointDTO, String> endpointsToChange = allEndpoints.stream()
+ .filter(e -> ymlDTOS.containsKey(e.getName()))
+ .filter(e -> !e.getName().equals("Local"))
+ .collect(Collectors.toMap(e -> e, e -> ymlDTOS.get(e.getName()).getYmlString()));
+
+ endpointsToChange.forEach((endpoint, ymlString) -> {
+ log.info("Trying to write {}, for external endpoint : {} , for user: {}",
+ name, endpoint.getName(), userInfo.getSimpleName());
+ String url = endpoint.getUrl() + "/api/configuration/" + findMethodName(name);
+ externalSelfService.post(url, ymlString, userInfo.getAccessToken(), String.class);
+ });
+ if (ymlDTOS.containsKey("Local")) {
+ overwriteProperties(path, name, ymlDTOS.get("Local").getYmlString());
+ }
}
- @Audit(action = RECONFIGURE, type = EDGE_NODE)
- public static void overwriteBillingServiceProperties(String ymlString) {
- writeFileFromString(ymlString, BILLING_SERVICE, BILLING_SERVICE_PROP_PATH);
+ private static String findMethodName(String name) {
+ switch (name) {
+ case "self-service.yml": {
+ return "self-service";
+ }
+ case "provisioning.yml": {
+ return "provisioning-service";
+ }
+ case "billing.yml": {
+ return "billing";
+ }
+ default:
+ return "";
+ }
}
public static void restart(boolean billing, boolean provserv, boolean ui) {
@@ -130,6 +161,14 @@ public class DynamicChangeProperties {
}
}
+ private static String readFileAsString(String selfServicePropPath, String serviceName, EndpointDTO endpoint, UserInfo userInfo) {
+ log.info("Trying to read self-service.yml, for external endpoint : {} , for user: {}",
+ endpoint, userInfo.getSimpleName());
+ String currentConf = externalSelfService.get(endpoint.getUrl() + "/api/admin/self-service",
+ userInfo.getAccessToken(), String.class);
+ return hideSecretsAndRemoveLicence(currentConf);
+ }
+
private static String hideSecretsAndRemoveLicence(String currentConf) {
Matcher m = Pattern.compile(SECRET_REGEX).matcher(currentConf);
List<String> secrets = new ArrayList<>();
@@ -164,7 +203,6 @@ public class DynamicChangeProperties {
log.error("Failed during overwriting {}", serviceName);
throw new DynamicChangePropertiesException(String.format("Failed during overwriting %s", serviceName));
}
-
}
private static void changeCHMODE(String serviceName, String path, String fromMode, String toMode) throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datalab.apache.org
For additional commands, e-mail: commits-help@datalab.apache.org