You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2021/04/29 06:09:47 UTC
[atlas] branch branch-2.0 updated: ATLAS-4261: Bulk Glossary Import
Response and Failed Error Message Improvements
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 6a02504 ATLAS-4261: Bulk Glossary Import Response and Failed Error Message Improvements
6a02504 is described below
commit 6a025043d2aac4d8e44bf0b3f5598bb3c766a38b
Author: sidmishra <si...@cloudera.com>
AuthorDate: Wed Apr 28 23:09:09 2021 -0700
ATLAS-4261: Bulk Glossary Import Response and Failed Error Message Improvements
Signed-off-by: Sarath Subramanian <sa...@apache.org>
(cherry picked from commit 7d7d1f3e1b1be9ea585398eb337208250730bda3)
---
.../main/java/org/apache/atlas/AtlasErrorCode.java | 4 +-
.../atlas/model/glossary/AtlasGlossaryTerm.java | 80 ++++++----
.../model/glossary/AtlasGlossaryTermHeader.java | 80 ++++++++++
.../org/apache/atlas/glossary/GlossaryService.java | 24 ++-
.../apache/atlas/glossary/GlossaryTermUtils.java | 171 +++++++++++++--------
5 files changed, 262 insertions(+), 97 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 5ef62d3..2febff4 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -208,6 +208,7 @@ public enum AtlasErrorCode {
GLOSSARY_ALREADY_EXISTS(409, "ATLAS-409-00-007", "Glossary with name {0} already exists"),
GLOSSARY_TERM_ALREADY_EXISTS(409, "ATLAS-409-00-009", "Glossary term with qualifiedName {0} already exists"),
GLOSSARY_CATEGORY_ALREADY_EXISTS(409, "ATLAS-409-00-00A", "Glossary category with qualifiedName {0} already exists"),
+ GLOSSARY_IMPORT_FAILED(409, "ATLAS-409-00-011", "Glossary import failed"),
// All internal errors go here
INTERNAL_ERROR(500, "ATLAS-500-00-001", "Internal server error {0}"),
@@ -230,7 +231,8 @@ public enum AtlasErrorCode {
DATA_ACCESS_LOAD_FAILED(500, "ATLAS-500-00-013", "Load failed: {0}"),
ENTITY_NOTIFICATION_FAILED(500, "ATLAS-500-00-014", "Notification failed for operation: {0} : {1}"),
FAILED_TO_UPLOAD(500, "ATLAS-500-00-015", "Error occurred while uploading the file: {0}"),
- FAILED_TO_CREATE_GLOSSARY_TERM(500, "ATLAS-500-00-016", "Error occurred while creating glossary term: {0}");
+ FAILED_TO_CREATE_GLOSSARY_TERM(500, "ATLAS-500-00-016", "Error occurred while creating glossary term: {0}"),
+ FAILED_TO_UPDATE_GLOSSARY_TERM(500, "ATLAS-500-00-017", "Error occurred while updating glossary term: {0}");
private String errorCode;
private String errorMessage;
diff --git a/intg/src/main/java/org/apache/atlas/model/glossary/AtlasGlossaryTerm.java b/intg/src/main/java/org/apache/atlas/model/glossary/AtlasGlossaryTerm.java
index 4fa1538..ee306c6 100644
--- a/intg/src/main/java/org/apache/atlas/model/glossary/AtlasGlossaryTerm.java
+++ b/intg/src/main/java/org/apache/atlas/model/glossary/AtlasGlossaryTerm.java
@@ -176,7 +176,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setSeeAlso(final Set<AtlasRelatedTermHeader> seeAlso) {
this.seeAlso = seeAlso;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(seeAlso)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getSynonyms() {
@@ -185,7 +188,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setSynonyms(final Set<AtlasRelatedTermHeader> synonyms) {
this.synonyms = synonyms;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(synonyms)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getAntonyms() {
@@ -194,7 +200,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setAntonyms(final Set<AtlasRelatedTermHeader> antonyms) {
this.antonyms = antonyms;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(antonyms)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getPreferredTerms() {
@@ -203,7 +212,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setPreferredTerms(final Set<AtlasRelatedTermHeader> preferredTerms) {
this.preferredTerms = preferredTerms;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(preferredTerms)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getPreferredToTerms() {
@@ -220,7 +232,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setReplacementTerms(final Set<AtlasRelatedTermHeader> replacementTerms) {
this.replacementTerms = replacementTerms;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(replacementTerms)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getReplacedBy() {
@@ -229,7 +244,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setReplacedBy(final Set<AtlasRelatedTermHeader> replacedBy) {
this.replacedBy = replacedBy;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(replacedBy)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getTranslationTerms() {
@@ -238,7 +256,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setTranslationTerms(final Set<AtlasRelatedTermHeader> translationTerms) {
this.translationTerms = translationTerms;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(translationTerms)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getTranslatedTerms() {
@@ -247,7 +268,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setTranslatedTerms(final Set<AtlasRelatedTermHeader> translatedTerms) {
this.translatedTerms = translatedTerms;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(translatedTerms)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getIsA() {
@@ -256,7 +280,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setIsA(final Set<AtlasRelatedTermHeader> isA) {
this.isA = isA;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(isA)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getClassifies() {
@@ -265,7 +292,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setClassifies(final Set<AtlasRelatedTermHeader> classifies) {
this.classifies = classifies;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(classifies)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getValidValues() {
@@ -274,7 +304,10 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setValidValues(final Set<AtlasRelatedTermHeader> validValues) {
this.validValues = validValues;
- hasTerms = true;
+
+ if (CollectionUtils.isNotEmpty(validValues)) {
+ hasTerms = true;
+ }
}
public Set<AtlasRelatedTermHeader> getValidValuesFor() {
@@ -283,23 +316,14 @@ public class AtlasGlossaryTerm extends AtlasGlossaryBaseObject {
public void setValidValuesFor(final Set<AtlasRelatedTermHeader> validValuesFor) {
this.validValuesFor = validValuesFor;
- hasTerms = true;
- }
-
- public boolean containAnyRelation() {
- return (CollectionUtils.isNotEmpty(getTranslationTerms()) ||
- CollectionUtils.isNotEmpty(getValidValuesFor()) ||
- CollectionUtils.isNotEmpty(getSynonyms()) ||
- CollectionUtils.isNotEmpty(getReplacedBy()) ||
- CollectionUtils.isNotEmpty(getValidValues()) ||
- CollectionUtils.isNotEmpty(getReplacementTerms()) ||
- CollectionUtils.isNotEmpty(getSeeAlso()) ||
- CollectionUtils.isNotEmpty(getTranslatedTerms()) ||
- CollectionUtils.isNotEmpty(getIsA()) ||
- CollectionUtils.isNotEmpty(getAntonyms()) ||
- CollectionUtils.isNotEmpty(getClassifies()) ||
- CollectionUtils.isNotEmpty(getPreferredToTerms()) ||
- CollectionUtils.isNotEmpty(getPreferredTerms()));
+
+ if (CollectionUtils.isNotEmpty(validValuesFor)) {
+ hasTerms = true;
+ }
+ }
+
+ public AtlasGlossaryTermHeader getGlossaryTermHeader() {
+ return new AtlasGlossaryTermHeader(this.getGuid(), this.getQualifiedName());
}
@JsonIgnore
diff --git a/intg/src/main/java/org/apache/atlas/model/glossary/AtlasGlossaryTermHeader.java b/intg/src/main/java/org/apache/atlas/model/glossary/AtlasGlossaryTermHeader.java
new file mode 100644
index 0000000..65a5ce0
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/model/glossary/AtlasGlossaryTermHeader.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.model.glossary;
+
+import org.apache.atlas.model.annotation.AtlasJSON;
+
+import java.util.Objects;
+
+@AtlasJSON
+public class AtlasGlossaryTermHeader {
+ private String glossaryGuid;
+ private String qualifiedName;
+
+ public AtlasGlossaryTermHeader(String glossaryGuid) {
+ this.glossaryGuid = glossaryGuid;
+ }
+
+ public AtlasGlossaryTermHeader(String glossaryGuid, String qualifiedName) {
+ this.glossaryGuid = glossaryGuid;
+ this.qualifiedName = qualifiedName;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("AtlasGlossaryTermHeader{");
+ sb.append("glossaryGuid='").append(glossaryGuid).append('\'');
+ sb.append(", qualifiedName='").append(qualifiedName).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+
+ public AtlasGlossaryTermHeader() {
+ }
+
+ public String getGlossaryGuid() {
+ return glossaryGuid;
+ }
+
+ public void setGlossaryGuid(final String glossaryGuid) {
+ this.glossaryGuid = glossaryGuid;
+ }
+
+ public String getQualifiedName() {
+ return qualifiedName;
+ }
+
+ public void setQualifiedName(final String qualifiedName) {
+ this.qualifiedName = qualifiedName;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (!(o instanceof org.apache.atlas.model.glossary.AtlasGlossaryTermHeader)) return false;
+ final org.apache.atlas.model.glossary.AtlasGlossaryTermHeader that = (org.apache.atlas.model.glossary.AtlasGlossaryTermHeader) o;
+ return Objects.equals(glossaryGuid, that.glossaryGuid) &&
+ Objects.equals(qualifiedName, that.qualifiedName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(glossaryGuid, qualifiedName);
+ }
+
+}
diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java
index b12c0b8..be757ad 100644
--- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java
+++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java
@@ -37,6 +37,7 @@ import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.FileUtils;
+import org.apache.atlas.utils.AtlasJson;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -56,6 +57,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.FAILED;
+import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.SUCCESS;
import static org.apache.atlas.glossary.GlossaryUtils.getAtlasGlossaryCategorySkeleton;
import static org.apache.atlas.glossary.GlossaryUtils.getAtlasGlossaryTermSkeleton;
import static org.apache.atlas.glossary.GlossaryUtils.getGlossarySkeleton;
@@ -1125,42 +1127,44 @@ public class GlossaryService {
List<String[]> fileData = FileUtils.readFileData(fileName, inputStream);
- List<AtlasGlossaryTerm> glossaryTermsWithoutRelations = glossaryTermUtils.getGlossaryTermDataList(fileData, ret);
+ List<AtlasGlossaryTerm> glossaryTermsWithoutRelations = glossaryTermUtils.getGlossaryTermDataWithoutRelations(fileData, ret);
createGlossaryTerms(glossaryTermsWithoutRelations, ret);
- List<AtlasGlossaryTerm> glossaryTermsWithRelations = glossaryTermUtils.getGlossaryTermDataList(fileData, ret, true);
+ List<AtlasGlossaryTerm> glossaryTermsWithRelations = glossaryTermUtils.getGlossaryTermDataWithRelations(fileData, ret);
updateGlossaryTermsRelation(glossaryTermsWithRelations, ret);
return ret;
}
- private void createGlossaryTerms(List<AtlasGlossaryTerm> glossaryTerms, BulkImportResponse bulkImportResponse) {
+ private void createGlossaryTerms(List<AtlasGlossaryTerm> glossaryTerms, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
for (AtlasGlossaryTerm glossaryTerm : glossaryTerms) {
String glossaryTermName = glossaryTerm.getName();
String glossaryName = getGlossaryName(glossaryTerm);
try {
- createTerm(glossaryTerm);
+ AtlasGlossaryTerm createdTerm = createTerm(glossaryTerm);
- bulkImportResponse.addToSuccessImportInfoList(new ImportInfo(glossaryName, glossaryTermName));
+ bulkImportResponse.addToSuccessImportInfoList(new ImportInfo(glossaryName, glossaryTermName, SUCCESS, AtlasJson.toJson(createdTerm.getGlossaryTermHeader())));
} catch (AtlasBaseException e) {
LOG.error(AtlasErrorCode.FAILED_TO_CREATE_GLOSSARY_TERM.toString(), glossaryTermName, e);
bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, e.getMessage()));
}
}
+
+ checkForSuccessImports(bulkImportResponse);
}
private void updateGlossaryTermsRelation(List<AtlasGlossaryTerm> glossaryTerms, BulkImportResponse bulkImportResponse) {
for (AtlasGlossaryTerm glossaryTerm : glossaryTerms) {
- if (glossaryTerm.containAnyRelation()) {
+ if (glossaryTerm.hasTerms()) {
String glossaryTermName = glossaryTerm.getName();
String glossaryName = getGlossaryName(glossaryTerm);
try {
updateTerm(glossaryTerm, false);
} catch (AtlasBaseException e) {
- LOG.error(AtlasErrorCode.FAILED_TO_CREATE_GLOSSARY_TERM.toString(), glossaryTermName, e);
+ LOG.error(AtlasErrorCode.FAILED_TO_UPDATE_GLOSSARY_TERM.toString(), glossaryTermName, e);
bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, e.getMessage()));
}
@@ -1180,4 +1184,10 @@ public class GlossaryService {
return ret;
}
+
+ private void checkForSuccessImports(BulkImportResponse bulkImportResponse) throws AtlasBaseException {
+ if (CollectionUtils.isEmpty(bulkImportResponse.getSuccessImportInfoList())) {
+ throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_IMPORT_FAILED);
+ }
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
index f5789a7..dc39fd2 100644
--- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
+++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
@@ -39,6 +39,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.FileUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +62,9 @@ public class GlossaryTermUtils extends GlossaryUtils {
private static final Logger LOG = LoggerFactory.getLogger(GlossaryTermUtils.class);
private static final boolean DEBUG_ENABLED = LOG.isDebugEnabled();
+ private static final int INDEX_FOR_GLOSSARY_AT_RECORD = 0;
+ private static final int INDEX_FOR_TERM_AT_RECORD = 1;
+
Map<String, String> glossaryNameGuidCacheForImport = new HashMap<>();
protected GlossaryTermUtils(AtlasRelationshipStore relationshipStore, AtlasTypeRegistry typeRegistry, DataAccess dataAccess) {
@@ -535,11 +539,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
}
}
- protected List<AtlasGlossaryTerm> getGlossaryTermDataList(List<String[]> fileData, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
- return getGlossaryTermDataList(fileData, bulkImportResponse, false);
- }
-
- protected List<AtlasGlossaryTerm> getGlossaryTermDataList(List<String[]> fileData, BulkImportResponse bulkImportResponse, boolean processRelations) throws AtlasBaseException {
+ protected List<AtlasGlossaryTerm> getGlossaryTermDataWithoutRelations(List<String[]> fileData, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
List<AtlasGlossaryTerm> glossaryTerms = new ArrayList<>();
int rowCount = 1;
@@ -548,40 +548,27 @@ public class GlossaryTermUtils extends GlossaryUtils {
AtlasGlossaryTerm glossaryTerm = new AtlasGlossaryTerm();
String glossaryName = StringUtils.EMPTY;
- if ((record.length < 1) || StringUtils.isBlank(record[0])) {
+ if (ArrayUtils.isEmpty(record) || StringUtils.isBlank(record[INDEX_FOR_GLOSSARY_AT_RECORD])) {
LOG.error("The GlossaryName is blank for the record : ", Arrays.toString(record));
+
failedTermMsgs.add("The GlossaryName is blank for the record : " + Arrays.toString(record));
} else {
- String glossaryGuid;
- glossaryName = record[0];
+ glossaryName = record[INDEX_FOR_GLOSSARY_AT_RECORD];
- if (glossaryNameGuidCacheForImport.get(glossaryName) != null) {
- glossaryGuid = glossaryNameGuidCacheForImport.get(glossaryName);
-
- } else {
- AtlasVertex vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME, GlossaryUtils.ATLAS_GLOSSARY_TYPENAME + "." + QUALIFIED_NAME_ATTR, glossaryName);
-
- glossaryGuid = (vertex != null) ? AtlasGraphUtilsV2.getIdFromVertex(vertex) : null;
- }
+ String glossaryGuid = glossaryNameGuidCacheForImport.get(glossaryName);
- if (glossaryGuid == null) {
- if (GlossaryService.isNameInvalid(glossaryName)) {
- LOG.error("The provided Glossary Name is invalid : " + glossaryName);
- failedTermMsgs.add("The provided Glossary Name is invalid : " + glossaryName);
- } else {
- AtlasGlossary glossary = new AtlasGlossary();
- glossary.setQualifiedName(glossaryName);
- glossary.setName(glossaryName);
+ if (StringUtils.isEmpty(glossaryGuid)) {
+ glossaryGuid = getGlossaryGUIDFromGraphDB(glossaryName);
- glossary = dataAccess.save(glossary);
- glossaryGuid = glossary.getGuid();
+ if (StringUtils.isEmpty(glossaryGuid)) {
+ glossaryGuid = createGlossary(glossaryName, failedTermMsgs);
}
- }
- if (glossaryGuid != null) {
glossaryNameGuidCacheForImport.put(glossaryName, glossaryGuid);
+ }
- glossaryTerm = populateGlossaryTermObject(failedTermMsgs, record, glossaryGuid, processRelations);
+ if (StringUtils.isNotEmpty(glossaryGuid)) {
+ glossaryTerm = populateGlossaryTermObject(failedTermMsgs, record, glossaryGuid, false);
glossaryTerm.setQualifiedName(getGlossaryTermQualifiedName(glossaryTerm.getName(), glossaryGuid));
@@ -590,7 +577,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
}
if (failedTermMsgs.size() > 0) {
- String failedTermMsg = StringUtils.join(failedTermMsgs, "\n");
+ String failedTermMsg = StringUtils.join(failedTermMsgs, System.lineSeparator());
String glossaryTermName = glossaryTerm.getName();
bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, failedTermMsg, rowCount));
@@ -602,6 +589,40 @@ public class GlossaryTermUtils extends GlossaryUtils {
return glossaryTerms;
}
+ protected List<AtlasGlossaryTerm> getGlossaryTermDataWithRelations(List<String[]> fileData, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
+ List<AtlasGlossaryTerm> glossaryTerms = new ArrayList<>();
+ int rowCount = 1;
+
+ for (String[] record : fileData) {
+ List<String> failedTermMsgs = new ArrayList<>();
+
+ if (ArrayUtils.isNotEmpty(record) && StringUtils.isNotBlank(record[INDEX_FOR_GLOSSARY_AT_RECORD])) {
+ AtlasGlossaryTerm glossaryTerm = new AtlasGlossaryTerm();
+ String glossaryName = record[INDEX_FOR_GLOSSARY_AT_RECORD];
+ String glossaryGuid = glossaryNameGuidCacheForImport.get(glossaryName);
+
+ if (StringUtils.isNotEmpty(glossaryGuid)) {
+ glossaryTerm = populateGlossaryTermObject(failedTermMsgs, record, glossaryGuid, true);
+
+ glossaryTerm.setQualifiedName(getGlossaryTermQualifiedName(glossaryTerm.getName(), glossaryGuid));
+
+ glossaryTerms.add(glossaryTerm);
+ }
+
+ if (failedTermMsgs.size() > 0) {
+ String failedTermMsg = StringUtils.join(failedTermMsgs, System.lineSeparator());
+ String glossaryTermName = glossaryTerm.getName();
+
+ bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, failedTermMsg, rowCount));
+ }
+ }
+
+ rowCount++;
+ }
+
+ return glossaryTerms;
+ }
+
public static String getGlossaryTermHeaders() {
List<String> ret = new ArrayList<>();
@@ -630,7 +651,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
return String.join(", ", ret);
}
- protected Map getMapValue(String csvRecord, List<String> failedTermMsgs) {
+ protected Map getMapValue(String csvRecord, List<String> failedTermMsgs, boolean populateRelations) {
Map ret = null;
if (StringUtils.isNotBlank(csvRecord)) {
@@ -643,9 +664,9 @@ public class GlossaryTermUtils extends GlossaryUtils {
if ((recordArray.length % 2) == 0) {
ret.put(recordArray[0], recordArray[1]);
- } else {
- failedTermMsgs.add("\n" + "The Data in the uploaded file is incorrectly specified : " + csvRecord
- + "\n" + "AdditionalAttributes needs to be a key:value pair");
+ } else if (!populateRelations) {
+ failedTermMsgs.add("The Data in the uploaded file is incorrectly specified : " + csvRecord
+ + System.lineSeparator() + "AdditionalAttributes needs to be a key:value pair");
}
}
}
@@ -667,8 +688,9 @@ public class GlossaryTermUtils extends GlossaryUtils {
Set ret = null;
if (StringUtils.isNotBlank(csvRecord)) {
- ret = new HashSet();
- String csvRecordArray[] = csvRecord.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER);
+ ret = new HashSet();
+ String csvRecordArray[] = csvRecord.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER);
+
AtlasRelatedTermHeader relatedTermHeader;
for (String data : csvRecordArray) {
@@ -679,7 +701,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME,
GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME + invalidNameChars[1] + QUALIFIED_NAME_ATTR, dataArray[1] + invalidNameChars[0] + dataArray[0]);
} else {
- failedTermMsgs.add("\n" + "Either incorrect data specified for Term or Term does not exist : " +termName);
+ failedTermMsgs.add("Either incorrect data specified for Term or Term does not exist : " +termName);
}
if (vertex != null) {
@@ -688,8 +710,8 @@ public class GlossaryTermUtils extends GlossaryUtils {
relatedTermHeader.setTermGuid(glossaryTermGuid);
ret.add(relatedTermHeader);
} else {
- failedTermMsgs.add("\n" + "The provided Reference Glossary and TermName does not exist in the system " +
- dataArray[1] + "@" + dataArray[0] + " for record with TermName : " + termName + " and GlossaryName : " + glossaryName);
+ failedTermMsgs.add("The provided Reference " + dataArray[1] + "@" + dataArray[0] +
+ " does not exist at Atlas referred at record with TermName : " + termName + " and GlossaryName : " + glossaryName);
}
}
}
@@ -698,15 +720,21 @@ public class GlossaryTermUtils extends GlossaryUtils {
}
protected AtlasGlossaryTerm populateGlossaryTermObject(List<String> failedTermMsgList, String[] record, String glossaryGuid, boolean populateRelations) {
- AtlasGlossaryTerm ret = new AtlasGlossaryTerm();
- int i = 0;
int length = record.length;
+ int i = INDEX_FOR_TERM_AT_RECORD;
+ AtlasGlossaryTerm ret = new AtlasGlossaryTerm();
- ret.setName((length > ++i) ? record[i] : null);
+ if (length > i) {
+ ret.setName(record[i]);
+ }
- if (!StringUtils.isNotBlank(ret.getName())) {
- failedTermMsgList.add("\n" + "The TermName is blank for provided record: " + Arrays.toString(record));
+ if (StringUtils.isBlank(ret.getName())) {
+ if (!populateRelations) {
+ failedTermMsgList.add("The TermName is blank for provided record: " + Arrays.toString(record));
+ }
} else {
+ ret.setAnchor(new AtlasGlossaryHeader(glossaryGuid));
+
ret.setShortDescription((length > ++i) ? record[i] : null);
ret.setLongDescription((length > ++i) ? record[i] : null);
@@ -717,36 +745,32 @@ public class GlossaryTermUtils extends GlossaryUtils {
ret.setUsage((length > ++i) ? record[i] : null);
- ret.setAdditionalAttributes(((length > ++i) ? (Map<String, Object>) getMapValue(record[i], failedTermMsgList) : null));
-
- ret.setAnchor(new AtlasGlossaryHeader(glossaryGuid));
+ ret.setAdditionalAttributes(((length > ++i) ? (Map<String, Object>) getMapValue(record[i], failedTermMsgList, populateRelations) : null));
if (populateRelations) {
- ret.setTranslationTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
-
- ret.setValidValuesFor((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setTranslationTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setSynonyms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setValidValuesFor((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setReplacedBy((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setSynonyms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setValidValues((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setValidValues((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setReplacementTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setReplacementTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setSeeAlso((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setSeeAlso((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setTranslatedTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setTranslatedTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setIsA((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setIsA((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setAntonyms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setAntonyms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setClassifies((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setClassifies((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setPreferredToTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setPreferredToTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
- ret.setPreferredTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[0], failedTermMsgList) : null);
+ ret.setPreferredTerms((length > ++i) ? (Set<AtlasRelatedTermHeader>) getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
}
}
@@ -758,4 +782,29 @@ public class GlossaryTermUtils extends GlossaryUtils {
return glossaryTermName + "@" + glossary.getQualifiedName();
}
+
+ private String getGlossaryGUIDFromGraphDB(String glossaryName) {
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME, GlossaryUtils.ATLAS_GLOSSARY_TYPENAME + "." + QUALIFIED_NAME_ATTR, glossaryName);
+
+ return (vertex != null) ? AtlasGraphUtilsV2.getIdFromVertex(vertex) : null;
+ }
+
+ private String createGlossary(String glossaryName, List<String> failedTermMsgs) throws AtlasBaseException {
+ String ret = null;
+
+ if (GlossaryService.isNameInvalid(glossaryName)) {
+ LOG.error("The provided Glossary Name is invalid : " + glossaryName);
+
+ failedTermMsgs.add("The provided Glossary Name {" + glossaryName + "} is invalid : " + AtlasErrorCode.INVALID_DISPLAY_NAME.getFormattedErrorMessage());
+ } else {
+ AtlasGlossary glossary = new AtlasGlossary();
+ glossary.setQualifiedName(glossaryName);
+ glossary.setName(glossaryName);
+
+ glossary = dataAccess.save(glossary);
+ ret = glossary.getGuid();
+ }
+
+ return ret;
+ }
}