You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2018/10/08 22:19:22 UTC
atlas git commit: ATLAS-2798: Export & Import Audits.
Repository: atlas
Updated Branches:
refs/heads/master 871f02bb3 -> 06ff0752f
ATLAS-2798: Export & Import Audits.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/06ff0752
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/06ff0752
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/06ff0752
Branch: refs/heads/master
Commit: 06ff0752f8ea447b4c1f04a1461f6cd61b758b90
Parents: 871f02b
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Thu Aug 2 08:26:20 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Mon Oct 8 15:08:05 2018 -0700
----------------------------------------------------------------------
addons/models/0000-Area0/0010-base_model.json | 73 ++++++++++
.../model/impexp/ExportImportAuditEntry.java | 133 +++++++++++++++++++
.../impexp/ExportImportAuditService.java | 121 +++++++++++++++++
.../ogm/AbstractDataTransferObject.java | 9 +-
.../apache/atlas/repository/ogm/DataAccess.java | 9 +-
.../ogm/ExportImportAuditEntryDTO.java | 96 +++++++++++++
.../test/java/org/apache/atlas/TestModules.java | 2 +
.../impexp/ExportImportAuditServiceTest.java | 118 ++++++++++++++++
.../migration/ComplexAttributesTest.java | 2 +-
.../migration/HiveParititionTest.java | 2 +-
.../repository/migration/HiveStocksTest.java | 2 +-
.../atlas/repository/migration/PathTest.java | 2 +-
.../migration/TypesWithClassificationTest.java | 2 +-
.../TypesWithCollectionsFinderTest.java | 2 +-
.../atlas/web/resources/AdminResource.java | 42 +++++-
.../atlas/web/resources/AdminResourceTest.java | 4 +-
16 files changed, 601 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/addons/models/0000-Area0/0010-base_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json
index 88ef0b2..edf055e 100644
--- a/addons/models/0000-Area0/0010-base_model.json
+++ b/addons/models/0000-Area0/0010-base_model.json
@@ -233,6 +233,79 @@
"isUnique": false
}
]
+ },
+ {
+ "name": "__ExportImportAuditEntry",
+ "typeVersion": "1.0",
+ "superTypes": [
+ "__internal"
+ ],
+ "attributeDefs": [
+ {
+ "name": "userName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "operation",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "sourceClusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "targetClusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "operationParams",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "operationStartTime",
+ "typeName": "long",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "operationEndTime",
+ "typeName": "long",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "resultSummary",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
}
],
"relationshipDefs": [
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/intg/src/main/java/org/apache/atlas/model/impexp/ExportImportAuditEntry.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/ExportImportAuditEntry.java b/intg/src/main/java/org/apache/atlas/model/impexp/ExportImportAuditEntry.java
new file mode 100644
index 0000000..a3d7a0e
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/ExportImportAuditEntry.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.model.impexp;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import org.apache.atlas.model.AtlasBaseModelObject;
+
+import java.io.Serializable;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ExportImportAuditEntry extends AtlasBaseModelObject implements Serializable {
+ private static final long serialVersionUID = 1L;
+ public static final String OPERATION_EXPORT = "EXPORT";
+ public static final String OPERATION_IMPORT = "IMPORT";
+
+ private String userName;
+ private String operation;
+ private String operationParams;
+ private long startTime;
+ private long endTime;
+ private String resultSummary;
+ private String sourceClusterName;
+ private String targetClusterName;
+
+ public ExportImportAuditEntry() {
+
+ }
+
+ public ExportImportAuditEntry(String sourceClusterName, String operation) {
+ this.sourceClusterName = sourceClusterName;
+ this.operation = operation;
+ }
+
+ public String getOperation() {
+ return operation;
+ }
+
+ public void setOperation(String operation) {
+ this.operation = operation;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getUserName() {
+ return this.userName;
+ }
+ public void setOperationParams(String operationParams) {
+ this.operationParams = operationParams;
+ }
+
+ public String getOperationParams() {
+ return this.operationParams;
+ }
+
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public void setEndTime(long endTime) {
+ this.endTime = endTime;
+ }
+
+ public long getEndTime() {
+ return this.endTime;
+ }
+
+ public String getTargetClusterName() {
+ return this.targetClusterName;
+ }
+
+ public String getSourceClusterName() {
+ return this.sourceClusterName;
+ }
+
+ public void setSourceClusterName(String sourceClusterName) {
+ this.sourceClusterName = sourceClusterName;
+ }
+
+ public void setTargetClusterName(String targetClusterName) {
+ this.targetClusterName = targetClusterName;
+ }
+
+ public String getResultSummary() {
+ return resultSummary;
+ }
+
+ public void setResultSummary(String resultSummary) {
+ this.resultSummary = resultSummary;
+ }
+
+ @Override
+ public StringBuilder toString(StringBuilder sb) {
+ sb.append(", userName: ").append(userName);
+ sb.append(", operation: ").append(operation);
+ sb.append(", operationParams: ").append(operationParams);
+ sb.append(", sourceClusterName: ").append(sourceClusterName);
+ sb.append(", targetClusterName: ").append(targetClusterName);
+ sb.append(", startTime: ").append(startTime);
+ sb.append(", endTime: ").append(endTime);
+ sb.append(", resultSummary: ").append(resultSummary);
+
+ return sb;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
new file mode 100644
index 0000000..ebfc33f
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportImportAuditService.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.annotation.AtlasService;
+import org.apache.atlas.discovery.AtlasDiscoveryService;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.impexp.ExportImportAuditEntry;
+import org.apache.atlas.repository.ogm.DataAccess;
+import org.apache.atlas.repository.ogm.ExportImportAuditEntryDTO;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+
+@AtlasService
+public class ExportImportAuditService {
+ private static final Logger LOG = LoggerFactory.getLogger(ExportImportAuditService.class);
+ private static final String ENTITY_TYPE_NAME = "__ExportImportAuditEntry";
+
+ private final DataAccess dataAccess;
+ private AtlasDiscoveryService discoveryService;
+
+ @Inject
+ public ExportImportAuditService(DataAccess dataAccess, AtlasDiscoveryService discoveryService) {
+ this.dataAccess = dataAccess;
+ this.discoveryService = discoveryService;
+ }
+
+ public void save(ExportImportAuditEntry entry) throws AtlasBaseException {
+ dataAccess.saveNoLoad(entry);
+ }
+ public ExportImportAuditEntry get(ExportImportAuditEntry entry) throws AtlasBaseException {
+ if(entry.getGuid() == null) {
+ throw new AtlasBaseException("entity does not have GUID set. load cannot proceed.");
+ }
+ return dataAccess.load(entry);
+ }
+
+ public AtlasSearchResult get(String userName, String operation, String sourceCluster, String targetCluster,
+ String startTime, String endTime,
+ int limit, int offset) throws AtlasBaseException {
+ SearchParameters.FilterCriteria criteria = new SearchParameters.FilterCriteria();
+ criteria.setCriterion(new ArrayList<SearchParameters.FilterCriteria>());
+
+ addSearchParameters(criteria, userName, operation, sourceCluster, targetCluster, startTime, endTime);
+
+ SearchParameters searchParameters = getSearchParameters(limit, offset, criteria);
+
+ return discoveryService.searchWithParameters(searchParameters);
+ }
+
+ private SearchParameters getSearchParameters(int limit, int offset, SearchParameters.FilterCriteria criteria) {
+ SearchParameters searchParameters = new SearchParameters();
+ searchParameters.setTypeName(ENTITY_TYPE_NAME);
+ searchParameters.setEntityFilters(criteria);
+ searchParameters.setLimit(limit);
+ searchParameters.setOffset(offset);
+ return searchParameters;
+ }
+
+ private void addSearchParameters(SearchParameters.FilterCriteria criteria,
+ String userName, String operation, String sourceCluster, String targetCluster,
+ String startTime, String endTime) {
+
+ addParameterIfValueNotEmpty(criteria, ExportImportAuditEntryDTO.PROPERTY_USER_NAME, userName);
+ addParameterIfValueNotEmpty(criteria, ExportImportAuditEntryDTO.PROPERTY_OPERATION, operation);
+ addParameterIfValueNotEmpty(criteria, ExportImportAuditEntryDTO.PROPERTY_SOURCE_CLUSTER_NAME, sourceCluster);
+ addParameterIfValueNotEmpty(criteria, ExportImportAuditEntryDTO.PROPERTY_TARGET_CLUSTER_NAME, targetCluster);
+ addParameterIfValueNotEmpty(criteria, ExportImportAuditEntryDTO.PROPERTY_START_TIME, startTime);
+ addParameterIfValueNotEmpty(criteria, ExportImportAuditEntryDTO.PROPERTY_END_TIME, endTime);
+ }
+
+ private void addParameterIfValueNotEmpty(SearchParameters.FilterCriteria criteria,
+ String attributeName, String value) {
+ if(StringUtils.isEmpty(value)) return;
+
+ boolean isFirstCriteria = criteria.getAttributeName() == null;
+ SearchParameters.FilterCriteria cx = isFirstCriteria
+ ? criteria
+ : new SearchParameters.FilterCriteria();
+
+ setCriteria(cx, attributeName, value);
+
+ if(isFirstCriteria) {
+ cx.setCondition(SearchParameters.FilterCriteria.Condition.AND);
+ }
+
+ if(!isFirstCriteria) {
+ criteria.getCriterion().add(cx);
+ }
+ }
+
+ private SearchParameters.FilterCriteria setCriteria(SearchParameters.FilterCriteria criteria, String attributeName, String value) {
+ criteria.setAttributeName(attributeName);
+ criteria.setAttributeValue(value);
+ criteria.setOperator(SearchParameters.Operator.EQ);
+
+ return criteria;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/main/java/org/apache/atlas/repository/ogm/AbstractDataTransferObject.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/ogm/AbstractDataTransferObject.java b/repository/src/main/java/org/apache/atlas/repository/ogm/AbstractDataTransferObject.java
index 1a0c0f7..36f458e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/ogm/AbstractDataTransferObject.java
+++ b/repository/src/main/java/org/apache/atlas/repository/ogm/AbstractDataTransferObject.java
@@ -19,7 +19,6 @@ package org.apache.atlas.repository.ogm;
import org.apache.atlas.model.AtlasBaseModelObject;
import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.repository.Constants;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang3.StringUtils;
@@ -30,16 +29,16 @@ public abstract class AbstractDataTransferObject<T extends AtlasBaseModelObject>
private final Class<T> objectType;
private final String entityTypeName;
+ protected AbstractDataTransferObject(AtlasTypeRegistry typeRegistry, Class<T> tClass) {
+ this(typeRegistry, tClass, tClass.getSimpleName());
+ }
+
protected AbstractDataTransferObject(AtlasTypeRegistry typeRegistry, Class<T> tClass, String entityTypeName) {
this.typeRegistry = typeRegistry;
this.objectType = tClass;
this.entityTypeName = entityTypeName;
}
- protected AbstractDataTransferObject(AtlasTypeRegistry typeRegistry, Class<T> tClass) {
- this(typeRegistry, tClass, Constants.INTERNAL_PROPERTY_KEY_PREFIX + tClass.getSimpleName());
- }
-
@Override
public Class getObjectType() {
return objectType;
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/main/java/org/apache/atlas/repository/ogm/DataAccess.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/ogm/DataAccess.java b/repository/src/main/java/org/apache/atlas/repository/ogm/DataAccess.java
index 6058867..bef7d05 100644
--- a/repository/src/main/java/org/apache/atlas/repository/ogm/DataAccess.java
+++ b/repository/src/main/java/org/apache/atlas/repository/ogm/DataAccess.java
@@ -53,6 +53,11 @@ public class DataAccess {
}
public <T extends AtlasBaseModelObject> T save(T obj) throws AtlasBaseException {
+ saveNoLoad(obj);
+ return this.load(obj);
+ }
+
+ public <T extends AtlasBaseModelObject> void saveNoLoad(T obj) throws AtlasBaseException {
Objects.requireNonNull(obj, "Can't save a null object");
AtlasPerfTracer perf = null;
@@ -78,13 +83,9 @@ public class DataAccess {
obj.setGuid(assignedGuid);
}
}
-
- return this.load(obj);
-
} finally {
AtlasPerfTracer.log(perf);
}
-
}
public <T extends AtlasBaseModelObject> Iterable<T> save(Iterable<T> obj) throws AtlasBaseException {
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/main/java/org/apache/atlas/repository/ogm/ExportImportAuditEntryDTO.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/ogm/ExportImportAuditEntryDTO.java b/repository/src/main/java/org/apache/atlas/repository/ogm/ExportImportAuditEntryDTO.java
new file mode 100644
index 0000000..c22d41f
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/ogm/ExportImportAuditEntryDTO.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.ogm;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.impexp.ExportImportAuditEntry;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.util.Map;
+
+@Component
+public class ExportImportAuditEntryDTO extends AbstractDataTransferObject<ExportImportAuditEntry> {
+
+ public static final String PROPERTY_USER_NAME = "userName";
+ public static final String PROPERTY_OPERATION = "operation";
+ public static final String PROPERTY_OPERATION_PARAMS = "operationParams";
+ public static final String PROPERTY_START_TIME = "operationStartTime";
+ public static final String PROPERTY_END_TIME = "operationEndTime";
+ public static final String PROPERTY_RESULT_SUMMARY = "resultSummary";
+ public static final String PROPERTY_SOURCE_CLUSTER_NAME = "sourceClusterName";
+ public static final String PROPERTY_TARGET_CLUSTER_NAME = "targetClusterName";
+
+ @Inject
+ public ExportImportAuditEntryDTO(AtlasTypeRegistry typeRegistry) {
+ super(typeRegistry, ExportImportAuditEntry.class,
+ Constants.INTERNAL_PROPERTY_KEY_PREFIX + ExportImportAuditEntry.class.getSimpleName());
+ }
+
+ @Override
+ public ExportImportAuditEntry from(AtlasEntity entity) {
+ ExportImportAuditEntry entry = new ExportImportAuditEntry();
+
+ setGuid(entry, entity);
+ entry.setUserName((String) entity.getAttribute(PROPERTY_USER_NAME));
+ entry.setOperation((String) entity.getAttribute(PROPERTY_OPERATION));
+ entry.setOperationParams((String) entity.getAttribute(PROPERTY_OPERATION_PARAMS));
+ entry.setStartTime((long) entity.getAttribute(PROPERTY_START_TIME));
+ entry.setEndTime((long) entity.getAttribute(PROPERTY_END_TIME));
+ entry.setSourceClusterName((String) entity.getAttribute(PROPERTY_SOURCE_CLUSTER_NAME));
+ entry.setTargetClusterName((String) entity.getAttribute(PROPERTY_TARGET_CLUSTER_NAME));
+ entry.setResultSummary((String) entity.getAttribute(PROPERTY_RESULT_SUMMARY));
+
+ return entry;
+ }
+
+ @Override
+ public ExportImportAuditEntry from(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
+ return from(entityWithExtInfo.getEntity());
+ }
+
+ @Override
+ public AtlasEntity toEntity(ExportImportAuditEntry obj) {
+ AtlasEntity entity = getDefaultAtlasEntity(obj);
+
+ entity.setAttribute(PROPERTY_USER_NAME, obj.getUserName());
+ entity.setAttribute(PROPERTY_OPERATION, obj.getOperation());
+ entity.setAttribute(PROPERTY_OPERATION_PARAMS, obj.getOperationParams());
+ entity.setAttribute(PROPERTY_START_TIME, obj.getStartTime());
+ entity.setAttribute(PROPERTY_END_TIME, obj.getEndTime());
+ entity.setAttribute(PROPERTY_SOURCE_CLUSTER_NAME, obj.getSourceClusterName());
+ entity.setAttribute(PROPERTY_TARGET_CLUSTER_NAME, obj.getTargetClusterName());
+ entity.setAttribute(PROPERTY_RESULT_SUMMARY, obj.getResultSummary());
+
+ return entity;
+ }
+
+ @Override
+ public AtlasEntity.AtlasEntityWithExtInfo toEntityWithExtInfo(ExportImportAuditEntry obj) throws AtlasBaseException {
+ return new AtlasEntity.AtlasEntityWithExtInfo(toEntity(obj));
+ }
+
+ @Override
+ public Map<String, Object> getUniqueAttributes(final ExportImportAuditEntry obj) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/test/java/org/apache/atlas/TestModules.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java b/repository/src/test/java/org/apache/atlas/TestModules.java
index b5e0871..4549d6b 100644
--- a/repository/src/test/java/org/apache/atlas/TestModules.java
+++ b/repository/src/test/java/org/apache/atlas/TestModules.java
@@ -42,6 +42,7 @@ import org.apache.atlas.repository.graphdb.GraphDBMigrator;
import org.apache.atlas.repository.graphdb.janus.migration.GraphDBGraphSONMigrator;
import org.apache.atlas.repository.impexp.ExportService;
import org.apache.atlas.repository.ogm.AtlasClusterDTO;
+import org.apache.atlas.repository.ogm.ExportImportAuditEntryDTO;
import org.apache.atlas.repository.ogm.profiles.AtlasSavedSearchDTO;
import org.apache.atlas.repository.ogm.profiles.AtlasUserProfileDTO;
import org.apache.atlas.repository.ogm.DTORegistry;
@@ -172,6 +173,7 @@ public class TestModules {
availableDTOs.addBinding().to(AtlasGlossaryTermDTO.class);
availableDTOs.addBinding().to(AtlasGlossaryCategoryDTO.class);
availableDTOs.addBinding().to(AtlasClusterDTO.class);
+ availableDTOs.addBinding().to(ExportImportAuditEntryDTO.class);
bind(DTORegistry.class).asEagerSingleton();
bind(DataAccess.class).asEagerSingleton();
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
new file mode 100644
index 0000000..626e6a7
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.model.impexp.ExportImportAuditEntry;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.io.IOException;
+
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertNotNull;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class ExportImportAuditServiceTest {
+ @Inject
+ AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ ExportImportAuditService auditService;
+
+ @BeforeClass
+ public void setup() throws IOException, AtlasBaseException {
+ loadBaseModel(typeDefStore, typeRegistry);
+ }
+
+ @Test
+ public void checkTypeRegistered() throws AtlasBaseException {
+ AtlasType auditEntryType = typeRegistry.getType("__" + ExportImportAuditEntry.class.getSimpleName());
+ assertNotNull(auditEntryType);
+ }
+
+ @Test
+ public void saveLogEntry() throws AtlasBaseException, InterruptedException {
+ final String source1 = "clx";
+ final String target1 = "cly";
+ ExportImportAuditEntry entry = saveAndGet(source1, ExportImportAuditEntry.OPERATION_EXPORT, target1);
+
+ String source2 = "clx2";
+ String target2 = "clx1";
+ ExportImportAuditEntry entry2 = saveAndGet(source2, ExportImportAuditEntry.OPERATION_EXPORT, target2);
+
+ Thread.sleep(1000);
+ ExportImportAuditEntry actualEntry = retrieveEntry(entry);
+ ExportImportAuditEntry actualEntry2 = retrieveEntry(entry2);
+
+ assertNotEquals(actualEntry.getGuid(), actualEntry2.getGuid());
+ assertNotNull(actualEntry.getGuid());
+ assertEquals(actualEntry.getSourceClusterName(), entry.getSourceClusterName());
+ assertEquals(actualEntry.getTargetClusterName(), entry.getTargetClusterName());
+ assertEquals(actualEntry.getOperation(), entry.getOperation());
+ }
+
+ @Test
+ public void numberOfSavedEntries_Retrieved() throws AtlasBaseException, InterruptedException {
+ final String source1 = "cluster1";
+ final String target1 = "cly";
+ int MAX_ENTRIES = 5;
+
+ for (int i = 0; i < MAX_ENTRIES; i++) {
+ saveAndGet(source1, ExportImportAuditEntry.OPERATION_EXPORT, target1);
+ }
+
+ Thread.sleep(5000);
+ AtlasSearchResult results = auditService.get(source1, ExportImportAuditEntry.OPERATION_EXPORT, "", "", "", "", 10, 0);
+ assertEquals(results.getEntities().size(), MAX_ENTRIES);
+ }
+
+
+ private ExportImportAuditEntry retrieveEntry(ExportImportAuditEntry entry) throws AtlasBaseException {
+ AtlasSearchResult result = auditService.get(entry.getUserName(), entry.getOperation(), entry.getSourceClusterName(),
+ entry.getTargetClusterName(), Long.toString(entry.getStartTime()), "", 10, 0);
+ assertNotNull(result);
+ assertEquals(result.getEntities().size(), 1);
+ entry.setGuid(result.getEntities().get(0).getGuid());
+ return auditService.get(entry);
+ }
+
+ private ExportImportAuditEntry saveAndGet(String sourceClusterName, String operation, String targetClusterName) throws AtlasBaseException {
+ ExportImportAuditEntry entry = new ExportImportAuditEntry(sourceClusterName, operation);
+
+ entry.setTargetClusterName(targetClusterName);
+ entry.setUserName("default");
+ entry.setStartTime(System.currentTimeMillis());
+ entry.setEndTime(System.currentTimeMillis() + 1000L);
+ auditService.save(entry);
+ return entry;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
index 4cf5050..f1e6063 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
@@ -43,7 +43,7 @@ public class ComplexAttributesTest extends MigrationBaseAsserts {
String ENTITY_TYPE = "entity_type";
String ENTITY_WITH_COMPLEX_COLL_TYPE = "entity_with_complex_collection_attr";
- final int EXPECTED_TOTAL_COUNT = 215;
+ final int EXPECTED_TOTAL_COUNT = 216;
final int EXPECTED_ENTITY_TYPE_COUNT = 16;
final int EXPECTED_STRUCT_TYPE_COUNT = 3;
final int EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT = 1;
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
index c33abff..7567daa 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
@@ -40,7 +40,7 @@ public class HiveParititionTest extends MigrationBaseAsserts {
@Test
public void fileImporterTest() throws IOException, AtlasBaseException {
- final int EXPECTED_TOTAL_COUNT = 142;
+ final int EXPECTED_TOTAL_COUNT = 144;
final int EXPECTED_DB_COUNT = 1;
final int EXPECTED_TABLE_COUNT = 2;
final int EXPECTED_COLUMN_COUNT = 7;
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
index 568ad29..35987d9 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
@@ -38,7 +38,7 @@ public class HiveStocksTest extends MigrationBaseAsserts {
@Test
public void migrateStocks() throws AtlasBaseException, IOException {
- final int EXPECTED_TOTAL_COUNT = 189;
+ final int EXPECTED_TOTAL_COUNT = 190;
final int EXPECTED_DB_COUNT = 1;
final int EXPECTED_TABLE_COUNT = 1;
final int EXPECTED_COLUMN_COUNT = 7;
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/test/java/org/apache/atlas/repository/migration/PathTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/PathTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/PathTest.java
index 1b26ed2..cd01c70 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/PathTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/PathTest.java
@@ -46,7 +46,7 @@ public class PathTest extends MigrationBaseAsserts {
@Test
public void migrationImport() throws IOException, AtlasBaseException {
- final int EXPECTED_TOTAL_COUNT = 90;
+ final int EXPECTED_TOTAL_COUNT = 92;
runFileImporter("path_db");
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithClassificationTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithClassificationTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithClassificationTest.java
index 2bc99db..653144d 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithClassificationTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithClassificationTest.java
@@ -39,7 +39,7 @@ public class TypesWithClassificationTest extends MigrationBaseAsserts {
@Test
public void verify() throws IOException, AtlasBaseException {
- int EXPECTED_TOTAL_COUNT = 60;
+ int EXPECTED_TOTAL_COUNT = 62;
String ENTITY_TYPE = "ComplexTraitType";
String LEGACY_TYPE_TRAIT = "legacy_traitprayivofx4";
String LEGACY_TYPE_VENDOR_PII = "legacy_VENDOR_PII";
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
index dad5e4a..2621dfd 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
@@ -56,7 +56,7 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts {
public void fetchAll() {
Map<String, Map<String, List<String>>> typeAttrMap = TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry);
- assertEquals(typeAttrMap.size(), 10);
+ assertEquals(typeAttrMap.size(), 11);
assertProperties(typeAttrMap, "__AtlasUserProfile", "ARRAY", "__AtlasUserProfile.savedSearches");
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index c957bc8..66d17dc 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -28,13 +28,24 @@ import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.impexp.*;
import org.apache.atlas.model.metrics.AtlasMetrics;
+import org.apache.atlas.repository.impexp.ExportService;
+import org.apache.atlas.repository.impexp.ImportService;
+import org.apache.atlas.repository.impexp.ZipSink;
+import org.apache.atlas.repository.impexp.ZipSource;
+import org.apache.atlas.repository.impexp.ExportImportAuditService;
import org.apache.atlas.repository.impexp.*;
import org.apache.atlas.services.MetricsService;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.SearchTracker;
+import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.atlas.utils.AtlasJson;
import org.apache.atlas.web.filters.AtlasCSRFPreventionFilter;
import org.apache.atlas.web.service.ServiceState;
@@ -81,6 +92,7 @@ import java.util.concurrent.locks.ReentrantLock;
@Service
public class AdminResource {
private static final Logger LOG = LoggerFactory.getLogger(AdminResource.class);
+ private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("AdminResource");
private static final String isCSRF_ENABLED = "atlas.rest-csrf.enabled";
private static final String BROWSER_USER_AGENT_PARAM = "atlas.rest-csrf.browser-useragents-regex";
@@ -109,6 +121,7 @@ public class AdminResource {
private final AtlasTypeRegistry typeRegistry;
private final MigrationProgressService migrationProgressService;
private final ReentrantLock importExportOperationLock;
+ private ExportImportAuditService exportImportAuditService;
static {
try {
@@ -121,7 +134,8 @@ public class AdminResource {
@Inject
public AdminResource(ServiceState serviceState, MetricsService metricsService, AtlasTypeRegistry typeRegistry,
ExportService exportService, ImportService importService, SearchTracker activeSearches,
- MigrationProgressService migrationProgressService) {
+ MigrationProgressService migrationProgressService,
+ ExportImportAuditService exportImportAuditService) {
this.serviceState = serviceState;
this.metricsService = metricsService;
this.exportService = exportService;
@@ -129,6 +143,7 @@ public class AdminResource {
this.activeSearches = activeSearches;
this.typeRegistry = typeRegistry;
this.migrationProgressService = migrationProgressService;
+ this.exportImportAuditService = exportImportAuditService;
importExportOperationLock = new ReentrantLock();
}
@@ -422,6 +437,31 @@ public class AdminResource {
}
@GET
+ @Path("/expimp/audit")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public AtlasSearchResult getExportImportAudit(@QueryParam("sourceClusterName") String sourceCluster,
+ @QueryParam("targetCluster") String targetCluster,
+ @QueryParam("userName") String userName,
+ @QueryParam("operation") String operation,
+ @QueryParam("startTime") String startTime,
+ @QueryParam("endTime") String endTime,
+ @QueryParam("limit") int limit,
+ @QueryParam("offset") int offset) throws AtlasBaseException {
+ AtlasPerfTracer perf = null;
+
+ try {
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "getExportImportAudit(" + sourceCluster + ")");
+ }
+
+ return exportImportAuditService.get(userName, operation, sourceCluster, targetCluster, startTime, endTime, limit, offset);
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+ }
+
+ @GET
@Path("activeSearches")
@Produces(Servlets.JSON_MEDIA_TYPE)
public Set<String> getActiveSearches() {
http://git-wip-us.apache.org/repos/asf/atlas/blob/06ff0752/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
index 58ea628..3bb1e31 100644
--- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
@@ -51,7 +51,7 @@ public class AdminResourceTest {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE);
- AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null);
+ AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null, null);
Response response = adminResource.getStatus();
assertEquals(response.getStatus(), HttpServletResponse.SC_OK);
JsonNode entity = AtlasJson.parseToV1JsonNode((String) response.getEntity());
@@ -62,7 +62,7 @@ public class AdminResourceTest {
public void testResourceGetsValueFromServiceState() throws IOException {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE);
- AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null);
+ AdminResource adminResource = new AdminResource(serviceState, null, null, null, null, null, null, null);
Response response = adminResource.getStatus();
verify(serviceState).getState();