You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by pi...@apache.org on 2024/06/25 05:51:52 UTC
(atlas) branch branch-2.0 updated: ATLAS-4866: Export/Import: Skip adding processed guids in audit while importing huge zip
This is an automated email from the ASF dual-hosted git repository.
pinal pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new e1a293195 ATLAS-4866: Export/Import: Skip adding processed guids in audit while importing huge zip
e1a293195 is described below
commit e1a2931959f7239daa905177f0cf9f4833cc7738
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Tue Jun 25 10:44:00 2024 +0530
ATLAS-4866: Export/Import: Skip adding processed guids in audit while importing huge zip
Signed-off-by: Pinal Shah <pi...@freestoneinfotech.com>
---
.../impexp/ExportImportAuditService.java | 28 +++++++++++++++++++++-
1 file changed, 27 insertions(+), 1 deletion(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
index 3afa17301..67340f8a8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
@@ -24,16 +24,20 @@ import org.apache.atlas.discovery.AtlasDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.impexp.ExportImportAuditEntry;
import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.ogm.DataAccess;
import org.apache.atlas.repository.ogm.ExportImportAuditEntryDTO;
+import org.apache.atlas.type.AtlasType;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;
import javax.inject.Inject;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -44,12 +48,14 @@ public class ExportImportAuditService {
private static final String ENTITY_TYPE_NAME = "__ExportImportAuditEntry";
private final DataAccess dataAccess;
+ private final EntityAuditRepository auditRepository;
private AtlasDiscoveryService discoveryService;
@Inject
- public ExportImportAuditService(DataAccess dataAccess, AtlasDiscoveryService discoveryService) {
+ public ExportImportAuditService(DataAccess dataAccess, AtlasDiscoveryService discoveryService, EntityAuditRepository auditRepository) {
this.dataAccess = dataAccess;
this.discoveryService = discoveryService;
+ this.auditRepository = auditRepository;
}
@GraphTransaction
@@ -164,6 +170,26 @@ public class ExportImportAuditService {
entry.setStartTime(startTime);
entry.setEndTime(endTime);
+ if (operation == ExportImportAuditEntry.OPERATION_IMPORT) {
+
+ String auditString = AtlasType.toJson(entry);
+ byte[] auditBytes = auditString.getBytes(StandardCharsets.UTF_8);
+ long auditSize = auditBytes != null ? auditBytes.length : 0;
+ long auditMaxSize = auditRepository.repositoryMaxSize();
+
+ if (auditMaxSize >= 0 && auditSize > auditMaxSize) {
+
+ AtlasImportResult importResult = AtlasType.fromJson(result, AtlasImportResult.class);
+ if (importResult != null && !CollectionUtils.isEmpty(importResult.getProcessedEntities())) {
+ LOG.warn("audit record too long: user={}, operation={}, size={}, maxSize={}, processedEntityCount={}, processed entities guids in resultSummary not stored in audit",
+ entry.getUserName(), entry.getOperation(), auditSize, auditMaxSize, importResult.getProcessedEntities().size());
+
+ importResult.getProcessedEntities().clear();
+ entry.setResultSummary(AtlasType.toJson(importResult));
+ }
+ }
+ }
+
save(entry);
LOG.info("addAuditEntry: user: {}, source: {}, target: {}, operation: {}", entry.getUserName(),
entry.getSourceServerName(), entry.getTargetServerName(), entry.getOperation());