You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/04/18 02:21:46 UTC
[inlong] branch master updated: [INLONG-7867][Manager] Support data validation when importing Excel file (#7869)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 57e95ea92 [INLONG-7867][Manager] Support data validation when importing Excel file (#7869)
57e95ea92 is described below
commit 57e95ea9205e7c2949a25b74f48265c41b6aabc7
Author: feat <fe...@outlook.com>
AuthorDate: Tue Apr 18 10:21:40 2023 +0800
[INLONG-7867][Manager] Support data validation when importing Excel file (#7869)
---
.../manager/common/tool/excel/ExcelTool.java | 65 +++++++++++++++++++---
.../tool/excel/validator/ExcelCellValidator.java | 12 ++++
.../excel/validator/NonEmptyCellValidator.java | 15 ++++-
.../inlong/manager/common/util/Preconditions.java | 9 ++-
.../pojo/stream/StreamFieldTypeCellValidator.java | 12 ++++
5 files changed, 103 insertions(+), 10 deletions(-)
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/ExcelTool.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/ExcelTool.java
index c88f1d584..4ceaeb523 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/ExcelTool.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/ExcelTool.java
@@ -22,6 +22,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.lang3.tuple.Triple;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
import org.apache.inlong.manager.common.tool.excel.annotation.ExcelEntity;
import org.apache.inlong.manager.common.tool.excel.annotation.ExcelField;
import org.apache.inlong.manager.common.tool.excel.annotation.Font;
@@ -372,35 +373,55 @@ public class ExcelTool {
int lastRowNum = sheet.getLastRowNum();
List<E> currentResult = new ArrayList<>(lastRowNum);
-
+ List<String> validateResult = new ArrayList<>(lastRowNum);
for (int rowNum = 1; rowNum <= lastRowNum; ++rowNum) {
XSSFRow row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
+ // Mark the row only if all cells are not null
+ Map<Integer, FieldMeta> positionFieldMetaMap = classMeta.positionFieldMetaMap;
+ boolean rowNotNull = positionFieldMetaMap.keySet()
+ .stream()
+ .map(colIndex -> row.getCell(colIndex,
+ Row.MissingCellPolicy.RETURN_BLANK_AS_NULL) != null)
+ .reduce(false, (left, right) -> left || right);
+
+ // Skip if all columns are null
+ if (!rowNotNull) {
+ continue;
+ }
+
E instance = null;
+ StringBuilder colValidateResult = new StringBuilder();
boolean hasValueInRow = false;
- for (Map.Entry<Integer, FieldMeta> entry : classMeta.positionFieldMetaMap.entrySet()) {
+ for (Map.Entry<Integer, FieldMeta> entry : positionFieldMetaMap.entrySet()) {
Integer colIndex = entry.getKey();
FieldMeta fieldMeta = entry.getValue();
XSSFCell cell = row.getCell(colIndex, Row.MissingCellPolicy.RETURN_BLANK_AS_NULL);
- if (cell == null) {
- continue;
- }
+
hasValueInRow = true;
ExcelCellDataTransfer cellDataTransfer = fieldMeta.getCellDataTransfer();
Object value = parseCellValue(cellDataTransfer, cell);
if (instance == null) {
instance = clazz.newInstance();
}
+ validateCellValue(fieldMeta, value).ifPresent(info -> colValidateResult.append("Column ")
+ .append(colIndex + 1).append(":").append(info).append(";"));
fieldMeta.getField().setAccessible(true);
fieldMeta.getField().set(instance, value);
-
}
if (hasValueInRow) {
currentResult.add(instance);
}
+ if (colValidateResult.length() > 0) {
+ String lineValidateResult =
+ String.format("Error in Row: %d, %s", (rowNum + 1), colValidateResult);
+ validateResult.add(lineValidateResult);
+ }
}
+ Preconditions.expectEmpty(validateResult, ErrorCodeEnum.INVALID_PARAMETER,
+ String.join("\n", validateResult));
result.addAll(currentResult);
}
}
@@ -457,6 +478,23 @@ public class ExcelTool {
return cellValue;
}
+ /**
+ * Validate the cell value of a given field in the Excel sheet
+ *
+ * @param fieldMeta the meta information of the field to validate
+ * @param value the value of the field to validate
+ */
+ private static Optional<String> validateCellValue(
+ FieldMeta fieldMeta,
+ Object value) {
+ ExcelCellValidator cellValidator = fieldMeta.getCellValidator();
+ if (cellValidator != null && !cellValidator.validate(value)) {
+ return Optional.of(cellValidator.getInvalidTip());
+ } else {
+ return Optional.empty();
+ }
+ }
+
@Data
static class FieldMeta implements Serializable {
@@ -480,6 +518,11 @@ public class ExcelTool {
*/
private ExcelCellDataTransfer cellDataTransfer;
+ /**
+ * The validator for the cell
+ */
+ private ExcelCellValidator<?> cellValidator;
+
/**
* The field object
*/
@@ -553,9 +596,14 @@ public class ExcelTool {
for (Field field : fields) {
ExcelField excelField = field.getAnnotation(ExcelField.class);
if (excelField != null) {
+ Class<? extends ExcelCellValidator> validatorClass = excelField.validator();
ExcelCellDataTransfer excelCellDataTransfer = excelField.x2oTransfer();
+ ExcelCellValidator excelCellValidator = null;
+ if (validatorClass != ExcelCellValidator.class) {
+ excelCellValidator = validatorClass.newInstance();
+ }
meta.addField(field.getName(), excelField.name(), field, field.getType(),
- excelCellDataTransfer);
+ excelCellDataTransfer, excelCellValidator);
}
}
@@ -563,7 +611,7 @@ public class ExcelTool {
}
private void addField(String fieldName, String excelName, Field field, Class<?> fieldType,
- ExcelCellDataTransfer cellDataTransfer) {
+ ExcelCellDataTransfer cellDataTransfer, ExcelCellValidator cellValidator) {
if (this.classFieldMetas == null) {
this.classFieldMetas = new ArrayList<>();
}
@@ -581,6 +629,7 @@ public class ExcelTool {
fieldMeta.setExcelName(excelName);
fieldMeta.setFieldType(fieldType);
fieldMeta.setCellDataTransfer(cellDataTransfer);
+ fieldMeta.setCellValidator(cellValidator);
fieldMeta.setField(field);
this.fieldNameMetaMap.put(fieldName, fieldMeta);
this.excelNameMetaMap.put(excelName, fieldMeta);
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/validator/ExcelCellValidator.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/validator/ExcelCellValidator.java
index ffd0234ff..3c2befc44 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/validator/ExcelCellValidator.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/validator/ExcelCellValidator.java
@@ -29,4 +29,16 @@ public interface ExcelCellValidator<T> extends Serializable {
* Returns the data validation constraint for the cell
*/
List<String> constraint();
+
+ /**
+ * Validates the cell value
+ * @param value the cell value to validate
+ * @return true if the value is valid, false otherwise
+ */
+ boolean validate(T value);
+
+ /**
+ * Returns the error message to display if the cell value is invalid
+ */
+ String getInvalidTip();
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/validator/NonEmptyCellValidator.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/validator/NonEmptyCellValidator.java
index 59cb4a22d..6710fbc01 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/validator/NonEmptyCellValidator.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/tool/excel/validator/NonEmptyCellValidator.java
@@ -17,6 +17,9 @@
package org.apache.inlong.manager.common.tool.excel.validator;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Collections;
import java.util.List;
/**
@@ -33,6 +36,16 @@ public class NonEmptyCellValidator implements ExcelCellValidator<String> {
*/
@Override
public List<String> constraint() {
- return null;
+ return Collections.emptyList();
+ }
+
+ @Override
+ public boolean validate(String value) {
+ return StringUtils.isNotBlank(value);
+ }
+
+ @Override
+ public String getInvalidTip() {
+ return "Value can not be empty!";
}
}
diff --git a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/util/Preconditions.java b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/util/Preconditions.java
index ce2ae4485..e27a2a162 100644
--- a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/util/Preconditions.java
+++ b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/util/Preconditions.java
@@ -17,6 +17,7 @@
package org.apache.inlong.manager.common.util;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
import org.apache.inlong.manager.common.exceptions.BusinessException;
@@ -24,6 +25,7 @@ import org.apache.inlong.manager.common.exceptions.BusinessException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -153,6 +155,12 @@ public class Preconditions {
}
}
+ public static void expectEmpty(List<String> obj, ErrorCodeEnum errorCodeEnum, String errMsg) {
+ if (CollectionUtils.isNotEmpty(obj)) {
+ throw new BusinessException(errorCodeEnum, errMsg);
+ }
+ }
+
public static void expectNotNull(Object obj, ErrorCodeEnum errorCodeEnum) {
if (obj == null) {
throw new BusinessException(errorCodeEnum);
@@ -191,5 +199,4 @@ public class Preconditions {
Set<String> set = new HashSet<>(Arrays.asList(separatedStr.split(separator)));
return set.contains(target);
}
-
}
diff --git a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/StreamFieldTypeCellValidator.java b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/StreamFieldTypeCellValidator.java
index 51e33e163..a5a49ac83 100644
--- a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/StreamFieldTypeCellValidator.java
+++ b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/StreamFieldTypeCellValidator.java
@@ -33,9 +33,21 @@ public class StreamFieldTypeCellValidator implements ExcelCellValidator<String>
// do nothing
}
+ private final String invalidateTip = String.format("StreamField type must be one of %s", STREAM_FIELD_TYPES);
+
@Override
public List<String> constraint() {
return new ArrayList<>(STREAM_FIELD_TYPES);
}
+ @Override
+ public boolean validate(String value) {
+ return STREAM_FIELD_TYPES.contains(value);
+ }
+
+ @Override
+ public String getInvalidTip() {
+ return invalidateTip;
+ }
+
}