You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by he...@apache.org on 2022/10/14 08:59:42 UTC
[inlong] branch master updated: [INLONG-6042][Manager] Support updating and deleting DataNode by key (#6043)
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 84a147ecc [INLONG-6042][Manager] Support updating and deleting DataNode by key (#6043)
84a147ecc is described below
commit 84a147ecca84f75188cf178b7947d66b9bbd78dc
Author: vernedeng <de...@pku.edu.cn>
AuthorDate: Fri Oct 14 16:59:37 2022 +0800
[INLONG-6042][Manager] Support updating and deleting DataNode by key (#6043)
* Support updating and deleting DataNode by key
* Improve the UpdateResult, and merge the error logs into one
Co-authored-by: healchow <he...@gmail.com>
---
.../client/api/inner/client/DataNodeClient.java | 31 ++++++++++++
.../manager/client/api/service/DataNodeApi.java | 8 +++
.../inlong/manager/pojo/common/UpdateResult.java | 46 +++++++++++++++++
.../service/node/AbstractDataNodeOperator.java | 4 +-
.../manager/service/node/DataNodeService.java | 20 ++++++++
.../manager/service/node/DataNodeServiceImpl.java | 45 ++++++++++++++++-
.../manager/web/controller/DataNodeController.java | 24 +++++++++
.../web/controller/DataNodeControllerTest.java | 57 ++++++++++++++++++++++
8 files changed, 232 insertions(+), 3 deletions(-)
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/DataNodeClient.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/DataNodeClient.java
index 1ca4a4b61..446be7512 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/DataNodeClient.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/DataNodeClient.java
@@ -23,6 +23,7 @@ import org.apache.inlong.manager.client.api.util.ClientUtils;
import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.pojo.common.PageResult;
import org.apache.inlong.manager.pojo.common.Response;
+import org.apache.inlong.manager.pojo.common.UpdateResult;
import org.apache.inlong.manager.pojo.node.DataNodeInfo;
import org.apache.inlong.manager.pojo.node.DataNodeRequest;
@@ -94,6 +95,21 @@ public class DataNodeClient {
return response.getData();
}
+ /**
+ * Update data node by unique key.
+ *
+ * @param request node info to be modified
+ * @return whether succeed
+ */
+ public UpdateResult updateByKey(DataNodeRequest request) {
+ Preconditions.checkNotNull(request, "request cannot be null");
+ Preconditions.checkNotEmpty(request.getName(), "data node name cannot be empty");
+ Preconditions.checkNotEmpty(request.getType(), "data node type cannot be empty");
+ Response<UpdateResult> response = ClientUtils.executeHttpCall(dataNodeApi.updateByKey(request));
+ ClientUtils.assertRespSuccess(response);
+ return response.getData();
+ }
+
/**
* Delete data node.
*
@@ -107,4 +123,19 @@ public class DataNodeClient {
return response.getData();
}
+ /**
+ * Delete data node by name and type.
+ *
+ * @param name node name to be deleted
+ * @param type node type to be deleted
+ * @return whether succeed
+ */
+ public Boolean deleteByKey(String name, String type) {
+ Preconditions.checkNotEmpty(name, "data node name cannot be empty or null");
+ Preconditions.checkNotEmpty(type, "data node type cannot be empty or null");
+ Response<Boolean> response = ClientUtils.executeHttpCall(dataNodeApi.deleteByKey(name, type));
+ ClientUtils.assertRespSuccess(response);
+ return response.getData();
+ }
+
}
diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/service/DataNodeApi.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/service/DataNodeApi.java
index eb16bd528..c94621363 100644
--- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/service/DataNodeApi.java
+++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/service/DataNodeApi.java
@@ -19,6 +19,7 @@ package org.apache.inlong.manager.client.api.service;
import org.apache.inlong.manager.pojo.common.PageResult;
import org.apache.inlong.manager.pojo.common.Response;
+import org.apache.inlong.manager.pojo.common.UpdateResult;
import org.apache.inlong.manager.pojo.node.DataNodeInfo;
import org.apache.inlong.manager.pojo.node.DataNodeRequest;
import retrofit2.Call;
@@ -27,6 +28,7 @@ import retrofit2.http.DELETE;
import retrofit2.http.GET;
import retrofit2.http.POST;
import retrofit2.http.Path;
+import retrofit2.http.Query;
public interface DataNodeApi {
@@ -42,7 +44,13 @@ public interface DataNodeApi {
@POST("node/update")
Call<Response<Boolean>> update(@Body DataNodeRequest request);
+ @POST("node/updateByKey")
+ Call<Response<UpdateResult>> updateByKey(@Body DataNodeRequest request);
+
@DELETE("node/delete/{id}")
Call<Response<Boolean>> delete(@Path("id") Integer id);
+ @DELETE("node/deleteByKey")
+ Call<Response<Boolean>> deleteByKey(@Query("name") String name, @Query("type") String type);
+
}
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/common/UpdateResult.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/common/UpdateResult.java
new file mode 100644
index 000000000..3235435d1
--- /dev/null
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/common/UpdateResult.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.pojo.common;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * Update result info
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@JsonIgnoreProperties(ignoreUnknown = true)
+@ApiModel("Update result info")
+public class UpdateResult {
+
+ @ApiModelProperty(value = "ID of the current record")
+ private Integer id;
+
+ @ApiModelProperty(value = "Update result, true or false")
+ private Boolean success;
+
+ @ApiModelProperty(value = "Version of the current record after updating")
+ private Integer version;
+
+}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/AbstractDataNodeOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/AbstractDataNodeOperator.java
index d66ff1233..0ccea9162 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/AbstractDataNodeOperator.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/AbstractDataNodeOperator.java
@@ -70,8 +70,8 @@ public abstract class AbstractDataNodeOperator implements DataNodeOperator {
entity.setModifier(operator);
int rowCount = dataNodeEntityMapper.updateByIdSelective(entity);
if (rowCount != InlongConstants.AFFECTED_ONE_ROW) {
- LOGGER.error("data node has already updated with name={}, type={}, curVersion={}", request.getName(),
- request.getType(), request.getVersion());
+ LOGGER.error("data node has already updated with name={}, type={}, request version={}, updated row={}",
+ request.getName(), request.getType(), request.getVersion(), rowCount);
throw new BusinessException(ErrorCodeEnum.CONFIG_EXPIRED);
}
}
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/DataNodeService.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/DataNodeService.java
index be5427097..35ef89b30 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/DataNodeService.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/DataNodeService.java
@@ -18,6 +18,7 @@
package org.apache.inlong.manager.service.node;
import org.apache.inlong.manager.pojo.common.PageResult;
+import org.apache.inlong.manager.pojo.common.UpdateResult;
import org.apache.inlong.manager.pojo.node.DataNodeInfo;
import org.apache.inlong.manager.pojo.node.DataNodePageRequest;
import org.apache.inlong.manager.pojo.node.DataNodeRequest;
@@ -61,6 +62,15 @@ public interface DataNodeService {
*/
Boolean update(DataNodeRequest request, String operator);
+ /**
+ * Update data node by key.
+ *
+ * @param request node info to be modified
+ * @param operator current operator
+ * @return Update result
+ */
+ UpdateResult updateByKey(DataNodeRequest request, String operator);
+
/**
* Delete data node.
*
@@ -70,6 +80,16 @@ public interface DataNodeService {
*/
Boolean delete(Integer id, String operator);
+ /**
+ * Delete data node by key.
+ *
+ * @param name node name to be deleted
+ * @param type node type to be deleted
+ * @param operator current operator
+ * @return whether succeed
+ */
+ Boolean deleteByKey(String name, String type, String operator);
+
/**
* Test whether the connection can be successfully established.
*
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/DataNodeServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/DataNodeServiceImpl.java
index aa823dae5..0a7073a6f 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/DataNodeServiceImpl.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/DataNodeServiceImpl.java
@@ -19,6 +19,7 @@ package org.apache.inlong.manager.service.node;
import com.github.pagehelper.Page;
import com.github.pagehelper.PageHelper;
+import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.consts.DataNodeType;
import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
@@ -27,6 +28,7 @@ import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.dao.entity.DataNodeEntity;
import org.apache.inlong.manager.dao.mapper.DataNodeEntityMapper;
import org.apache.inlong.manager.pojo.common.PageResult;
+import org.apache.inlong.manager.pojo.common.UpdateResult;
import org.apache.inlong.manager.pojo.node.DataNodeInfo;
import org.apache.inlong.manager.pojo.node.DataNodePageRequest;
import org.apache.inlong.manager.pojo.node.DataNodeRequest;
@@ -122,6 +124,7 @@ public class DataNodeServiceImpl implements DataNodeService {
LOGGER.error("data node not found by id={}", id);
throw new BusinessException(String.format("data node not found by id=%s", id));
}
+
String errMsg = String.format("data node has already updated with name=%s, type=%s, curVersion=%s",
entity.getName(), entity.getType(), request.getVersion());
if (!Objects.equals(entity.getVersion(), request.getVersion())) {
@@ -134,6 +137,32 @@ public class DataNodeServiceImpl implements DataNodeService {
return true;
}
+ @Override
+ public UpdateResult updateByKey(DataNodeRequest request, String operator) {
+ String name = request.getName();
+ String type = request.getType();
+ if (StringUtils.isEmpty(name) || StringUtils.isEmpty(type)) {
+ throw new BusinessException("data node name or type should not be empty or null");
+ }
+ DataNodeEntity entity = dataNodeMapper.selectByNameAndType(name, type);
+ if (entity == null) {
+ LOGGER.error("data node not found by name={}, type={}", name, type);
+ throw new BusinessException(String.format("data node not found by name=%s, type=%s", name, type));
+ }
+
+ request.setId(entity.getId());
+ String errMsg = String.format("data node has already updated with name=%s, type=%s, curVersion=%s",
+ entity.getName(), entity.getType(), request.getVersion());
+ if (!Objects.equals(entity.getVersion(), request.getVersion())) {
+ LOGGER.error(errMsg);
+ throw new BusinessException(ErrorCodeEnum.CONFIG_EXPIRED);
+ }
+ DataNodeOperator dataNodeOperator = operatorFactory.getInstance(request.getType());
+ dataNodeOperator.updateOpt(request, operator);
+ LOGGER.info("success to update data node={}", request);
+ return new UpdateResult(entity.getId(), true, request.getVersion() + 1);
+ }
+
@Override
public Boolean delete(Integer id, String operator) {
DataNodeEntity entity = dataNodeMapper.selectById(id);
@@ -142,6 +171,10 @@ public class DataNodeServiceImpl implements DataNodeService {
return false;
}
+ return delete(entity, operator);
+ }
+
+ private Boolean delete(DataNodeEntity entity, String operator) {
entity.setIsDeleted(entity.getId());
entity.setModifier(operator);
int rowCount = dataNodeMapper.updateById(entity);
@@ -150,10 +183,20 @@ public class DataNodeServiceImpl implements DataNodeService {
entity.getName(), entity.getType(), entity.getVersion());
throw new BusinessException(ErrorCodeEnum.CONFIG_EXPIRED);
}
- LOGGER.info("success to delete data node by id={}", id);
+ LOGGER.info("success to delete data node by id={}, name={}", entity.getId(), entity.getName());
return true;
}
+ @Override
+ public Boolean deleteByKey(String name, String type, String operator) {
+ DataNodeEntity entity = dataNodeMapper.selectByNameAndType(name, type);
+ if (entity == null || entity.getIsDeleted() > InlongConstants.UN_DELETED) {
+ LOGGER.error("data node not found or was already deleted for name={}", name);
+ return false;
+ }
+ return delete(entity, operator);
+ }
+
@Override
public Boolean testConnection(DataNodeRequest request) {
LOGGER.info("begin test connection for: {}", request);
diff --git a/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/DataNodeController.java b/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/DataNodeController.java
index bd2d81933..641abc9b9 100644
--- a/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/DataNodeController.java
+++ b/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/DataNodeController.java
@@ -19,11 +19,13 @@ package org.apache.inlong.manager.web.controller;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.apache.inlong.manager.common.enums.OperationType;
import org.apache.inlong.manager.common.validation.UpdateValidation;
import org.apache.inlong.manager.pojo.common.PageResult;
import org.apache.inlong.manager.pojo.common.Response;
+import org.apache.inlong.manager.pojo.common.UpdateResult;
import org.apache.inlong.manager.pojo.node.DataNodeInfo;
import org.apache.inlong.manager.pojo.node.DataNodePageRequest;
import org.apache.inlong.manager.pojo.node.DataNodeRequest;
@@ -40,6 +42,7 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
@@ -83,6 +86,14 @@ public class DataNodeController {
return Response.success(dataNodeService.update(request, username));
}
+ @PostMapping(value = "/node/updateByKey")
+ @OperationLog(operation = OperationType.UPDATE)
+ @ApiOperation(value = "Update data node by key")
+ public Response<UpdateResult> updateByKey(@RequestBody DataNodeRequest request) {
+ String username = LoginUserUtils.getLoginUser().getName();
+ return Response.success(dataNodeService.updateByKey(request, username));
+ }
+
@DeleteMapping(value = "/node/delete/{id}")
@ApiOperation(value = "Delete data node by id")
@OperationLog(operation = OperationType.DELETE)
@@ -92,6 +103,19 @@ public class DataNodeController {
return Response.success(dataNodeService.delete(id, LoginUserUtils.getLoginUser().getName()));
}
+ @DeleteMapping(value = "/node/deleteByKey")
+ @ApiOperation(value = "Delete data node by key")
+ @OperationLog(operation = OperationType.DELETE)
+ @ApiImplicitParams({
+ @ApiImplicitParam(name = "name", value = "Data node name", dataTypeClass = String.class, required = true),
+ @ApiImplicitParam(name = "type", value = "Data node type", dataTypeClass = String.class, required = true)
+ })
+ @RequiresRoles(value = UserRoleCode.ADMIN)
+ public Response<Boolean> deleteByKey(@RequestParam String name, @RequestParam String type) {
+ return Response.success(dataNodeService.deleteByKey(name, type,
+ LoginUserUtils.getLoginUser().getName()));
+ }
+
@PostMapping("/node/testConnection")
@ApiOperation(value = "Test connection for data node")
public Response<Boolean> testConnection(@Validated @RequestBody DataNodeRequest request) {
diff --git a/inlong-manager/manager-web/src/test/java/org/apache/inlong/manager/web/controller/DataNodeControllerTest.java b/inlong-manager/manager-web/src/test/java/org/apache/inlong/manager/web/controller/DataNodeControllerTest.java
index 7d1c74a2d..4f73f679e 100644
--- a/inlong-manager/manager-web/src/test/java/org/apache/inlong/manager/web/controller/DataNodeControllerTest.java
+++ b/inlong-manager/manager-web/src/test/java/org/apache/inlong/manager/web/controller/DataNodeControllerTest.java
@@ -22,6 +22,7 @@ import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.dao.entity.DataNodeEntity;
import org.apache.inlong.manager.dao.mapper.DataNodeEntityMapper;
import org.apache.inlong.manager.pojo.common.Response;
+import org.apache.inlong.manager.pojo.common.UpdateResult;
import org.apache.inlong.manager.pojo.node.DataNodeInfo;
import org.apache.inlong.manager.pojo.node.DataNodeRequest;
import org.apache.inlong.manager.pojo.node.hive.HiveDataNodeRequest;
@@ -85,6 +86,33 @@ class DataNodeControllerTest extends WebBaseTest {
Assertions.assertEquals(dataNodeEntity.getId(), dataNodeEntity.getIsDeleted());
}
+ @Test
+ void testSaveAndGetAndDeleteByKey() throws Exception {
+ HiveDataNodeRequest request = getHiveDataNodeRequest();
+ // save
+ MvcResult mvcResult = postForSuccessMvcResult("/api/node/save", request);
+
+ Integer dataNodeId = getResBodyObj(mvcResult, Integer.class);
+ Assertions.assertNotNull(dataNodeId);
+
+ // get
+ MvcResult getResult = getForSuccessMvcResult("/api/node/get/{id}", dataNodeId);
+
+ DataNodeInfo dataNode = getResBodyObj(getResult, DataNodeInfo.class);
+ Assertions.assertNotNull(dataNode);
+ Assertions.assertEquals(getHiveDataNodeRequest().getName(), dataNode.getName());
+
+ // delete
+ MvcResult deleteResult = deleteForSuccessMvcResult("/api/node/deleteByKey?name=" + request.getName()
+ + "&type=" + request.getType());
+
+ Boolean success = getResBodyObj(deleteResult, Boolean.class);
+ Assertions.assertTrue(success);
+
+ DataNodeEntity dataNodeEntity = dataNodeEntityMapper.selectById(dataNodeId);
+ Assertions.assertEquals(dataNodeEntity.getId(), dataNodeEntity.getIsDeleted());
+ }
+
@Test
void testUpdate() throws Exception {
// insert the test data
@@ -114,6 +142,35 @@ class DataNodeControllerTest extends WebBaseTest {
Assertions.assertEquals(request.getName(), dataNodeEntity.getName());
}
+ @Test
+ void testUpdateByKey() throws Exception {
+ // insert the test data
+ DataNodeEntity nodeEntity = new DataNodeEntity();
+ nodeEntity.setName("hiveNode1");
+ nodeEntity.setType(DataNodeType.HIVE);
+ nodeEntity.setIsDeleted(0);
+ nodeEntity.setModifier("test");
+ nodeEntity.setCreator("test");
+ nodeEntity.setUrl("old url");
+ nodeEntity.setCreateTime(new Date());
+ nodeEntity.setModifyTime(new Date());
+ nodeEntity.setInCharges("test");
+ nodeEntity.setVersion(InlongConstants.INITIAL_VERSION);
+
+ dataNodeEntityMapper.insert(nodeEntity);
+
+ DataNodeRequest request = getHiveDataNodeRequest();
+ request.setVersion(nodeEntity.getVersion());
+ MvcResult mvcResult = postForSuccessMvcResult("/api/node/updateByKey", request);
+
+ UpdateResult result = getResBodyObj(mvcResult, UpdateResult.class);
+ Assertions.assertTrue(result.getSuccess());
+ Assertions.assertEquals(request.getVersion() + 1, result.getVersion());
+
+ DataNodeEntity dataNodeEntity = dataNodeEntityMapper.selectByNameAndType(request.getName(), request.getType());
+ Assertions.assertEquals(request.getUrl(), dataNodeEntity.getUrl());
+ }
+
@Test
void testUpdateFailByNoId() throws Exception {
MvcResult mvcResult = postForSuccessMvcResult("/api/node/update", getHiveDataNodeRequest());