You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/03/17 07:35:47 UTC
[1/3] incubator-atlas git commit: ATLAS-1611: incorrect error code
for negative tests (#2)
Repository: incubator-atlas
Updated Branches:
refs/heads/0.8-incubating f454e7a44 -> 4baa23f1c
ATLAS-1611: incorrect error code for negative tests (#2)
(cherry picked from commit 89a387279d9d0cae66f8e0a75a9ee3610da8c173)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/2cf954a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/2cf954a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/2cf954a4
Branch: refs/heads/0.8-incubating
Commit: 2cf954a4994c1b3c089a2fd0b1166da8589b23b5
Parents: f454e7a
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Mar 10 14:36:14 2017 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Mar 17 00:35:00 2017 -0700
----------------------------------------------------------------------
intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 4 +++-
.../repository/converters/AtlasInstanceConverter.java | 11 ++++++++---
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2cf954a4/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 69c201d..5054cf0 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -67,6 +67,7 @@ public enum AtlasErrorCode {
INSTANCE_LINEAGE_INVALID_PARAMS(400, "ATLAS-400-00-026", "Invalid lineage query parameters passed {0}: {1}"),
ATTRIBUTE_UPDATE_NOT_SUPPORTED(400, "ATLAS-400-00-027", "{0}.{1} : attribute update not supported"),
INVALID_VALUE(400, "ATLAS-400-00-028", "invalid value: {0}"),
+ BAD_REQUEST(400, "ATLAS-400-00-020", "{0}"),
// All Not found enums go here
TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),
@@ -78,11 +79,12 @@ public enum AtlasErrorCode {
CLASSIFICATION_NOT_FOUND(404, "ATLAS-404-00-008", "Given classification {0} was invalid"),
INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND(404, "ATLAS-404-00-009", "Instance {0} with unique attribute {1} does not exist"),
REFERENCED_ENTITY_NOT_FOUND(404, "ATLAS-404-00-00A", "Referenced entity {0} is not found"),
+ INSTANCE_NOT_FOUND(404, "ATLAS-404-00-00B", "Given instance is invalid/not found: {0}"),
// All data conflict errors go here
TYPE_ALREADY_EXISTS(409, "ATLAS-409-00-001", "Given type {0} already exists"),
TYPE_HAS_REFERENCES(409, "ATLAS-409-00-002", "Given type {0} has references"),
- INSTANCE_ALREADY_EXISTS(409, "ATLAS-409-00-003", "Given entity {0} already exists"),
+ INSTANCE_ALREADY_EXISTS(409, "ATLAS-409-00-003", "failed to update entity: {0}"),
// All internal errors go here
INTERNAL_ERROR(500, "ATLAS-500-00-001", "Internal server error {0}"),
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2cf954a4/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
index a4f99a5..6e0766d 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
@@ -45,6 +45,7 @@ import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.exception.EntityExistsException;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.TraitNotFoundException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
@@ -199,19 +200,23 @@ public class AtlasInstanceConverter {
}
public static AtlasBaseException toAtlasBaseException(AtlasException e) {
+ if (e instanceof EntityExistsException) {
+ return new AtlasBaseException(AtlasErrorCode.INSTANCE_ALREADY_EXISTS, e.getMessage());
+ }
+
if ( e instanceof EntityNotFoundException || e instanceof TraitNotFoundException) {
- return new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, e);
+ return new AtlasBaseException(AtlasErrorCode.INSTANCE_NOT_FOUND, e.getMessage());
}
if ( e instanceof TypeNotFoundException) {
- return new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, e);
+ return new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, e.getMessage());
}
if (e instanceof ValueConversionException) {
return new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, e, e.getMessage());
}
- return new AtlasBaseException(e);
+ return new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, e.getMessage());
}
[3/3] incubator-atlas git commit: ATLAS-1630: basic search
performance improvement (#3)
Posted by ma...@apache.org.
ATLAS-1630: basic search performance improvement (#3)
(cherry picked from commit 0c9790289cf707a35b09c92edcd1484de8523c82)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4baa23f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4baa23f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4baa23f1
Branch: refs/heads/0.8-incubating
Commit: 4baa23f1c0d6c4bb82f1ceb610392440f6d9a474
Parents: 5b59acd
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Mar 10 16:43:42 2017 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Mar 17 00:35:34 2017 -0700
----------------------------------------------------------------------
.../apache/atlas/discovery/EntityDiscoveryService.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4baa23f1/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index 881c5a3..1cf4434 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -247,18 +247,18 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
final Map<String, Object> bindings = new HashMap<>();
String basicQuery = "g.V()";
- if (typeNames != null) {
- bindings.put("typeNames", typeNames);
-
- basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER);
- }
-
if (classificationNames != null) {
bindings.put("traitNames", classificationNames);
basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER);
}
+ if (typeNames != null) {
+ bindings.put("typeNames", typeNames);
+
+ basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER);
+ }
+
bindings.put("startIdx", params.offset());
bindings.put("endIdx", params.offset() + params.limit());
[2/3] incubator-atlas git commit: ATLAS-1503: optimization of export
implementation
Posted by ma...@apache.org.
ATLAS-1503: optimization of export implementation
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
(cherry picked from commit 7154e12d13fe4ccba4207bbf6f411353411f06b2)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/5b59acd7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/5b59acd7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/5b59acd7
Branch: refs/heads/0.8-incubating
Commit: 5b59acd7d3b435521b28e0a36968275750738132
Parents: 2cf954a
Author: ashutoshm <am...@hortonworks.com>
Authored: Fri Mar 10 10:50:51 2017 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Mar 17 00:35:16 2017 -0700
----------------------------------------------------------------------
.../atlas/model/impexp/AtlasExportResult.java | 20 +++++++
.../atlas/web/resources/AdminResource.java | 10 ++--
.../atlas/web/resources/ExportService.java | 62 +++++++++++++++-----
.../org/apache/atlas/web/resources/ZipSink.java | 18 +-----
4 files changed, 74 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b59acd7/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java
index e6a967e..8f3075e 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java
@@ -180,6 +180,16 @@ public class AtlasExportResult implements Serializable {
return toString(new StringBuilder()).toString();
}
+ public void clear() {
+ if(this.data != null) {
+ this.data.clear();
+ }
+
+ if(this.metrics != null) {
+ this.metrics.clear();
+ }
+ }
+
@JsonAutoDetect(getterVisibility=PUBLIC_ONLY, setterVisibility=PUBLIC_ONLY, fieldVisibility=NONE)
@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown=true)
@@ -234,5 +244,15 @@ public class AtlasExportResult implements Serializable {
public String toString() {
return toString(new StringBuilder()).toString();
}
+
+ public void clear() {
+ if(this.typesDef!= null) {
+ this.typesDef.clear();
+ }
+
+ if(this.entityCreationOrder != null) {
+ this.entityCreationOrder.clear();
+ }
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b59acd7/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index 31a4cf9..0dfdeb2 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -51,7 +51,6 @@ import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.context.SecurityContextHolder;
import javax.inject.Singleton;
-import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.*;
@@ -309,7 +308,7 @@ public class AdminResource {
ZipSink exportSink = null;
try {
- exportSink = new ZipSink();
+ exportSink = new ZipSink(httpServletResponse.getOutputStream());
ExportService exportService = new ExportService(this.typeRegistry);
AtlasExportResult result = exportService.run(exportSink, request, Servlets.getUserName(httpServletRequest),
@@ -318,14 +317,13 @@ public class AdminResource {
exportSink.close();
- ServletOutputStream outStream = httpServletResponse.getOutputStream();
- exportSink.writeTo(outStream);
-
+ httpServletResponse.addHeader("Content-Encoding","gzip");
httpServletResponse.setContentType("application/zip");
httpServletResponse.setHeader("Content-Disposition",
"attachment; filename=" + result.getClass().getSimpleName());
+ httpServletResponse.setHeader("Transfer-Encoding", "chunked");
- outStream.flush();
+ httpServletResponse.getOutputStream().flush();
return Response.ok().build();
} catch (IOException excp) {
LOG.error("export() failed", excp);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b59acd7/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java b/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java
index c1891e0..e123ff7 100644
--- a/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/ExportService.java
@@ -105,8 +105,9 @@ public class ExportService {
LOG.error("Operation failed: ", ex);
} finally {
atlasGraph.releaseGremlinScriptEngine(context.scriptEngine);
-
LOG.info("<== export(user={}, from={}): status {}", userName, requestingIP, context.result.getOperationStatus());
+ context.clear();
+ result.clear();
}
return context.result;
@@ -124,8 +125,8 @@ public class ExportService {
processEntity(entity, context, TraversalDirection.UNKNOWN);
}
- while (!context.guidsToProcess.isEmpty()) {
- String guid = context.guidsToProcess.remove(0);
+ while (!context.guidsToProcessIsEmpty()) {
+ String guid = context.guidsToProcessRemove(0);
TraversalDirection direction = context.guidDirection.get(guid);
AtlasEntity entity = entityGraphRetriever.toAtlasEntity(guid);
@@ -245,7 +246,7 @@ public class ExportService {
private void getEntityGuidsForConnectedFetch(AtlasEntity entity, ExportContext context, TraversalDirection direction) throws AtlasBaseException {
if (direction == TraversalDirection.UNKNOWN) {
- getConnectedEntityGuids(entity, context, TraversalDirection.OUTWARD, TraversalDirection.OUTWARD);
+ getConnectedEntityGuids(entity, context, TraversalDirection.OUTWARD, TraversalDirection.INWARD);
} else {
if (isProcessEntity(entity)) {
direction = TraversalDirection.OUTWARD;
@@ -271,7 +272,7 @@ public class ExportService {
String query = getQueryForTraversalDirection(direction);
if (LOG.isDebugEnabled()) {
- LOG.debug("==> getConnectedEntityGuids({}): guidsToProcess {} query {}", AtlasTypeUtil.getAtlasObjectId(entity), context.guidsToProcess.size(), query);
+ LOG.debug("==> getConnectedEntityGuids({}): guidsToProcess {} query {}", AtlasTypeUtil.getAtlasObjectId(entity), context.guidsToProcessSize(), query);
}
context.bindings.clear();
@@ -289,8 +290,8 @@ public class ExportService {
if (currentDirection == null) {
context.guidDirection.put(guid, direction);
- if (!context.guidsToProcess.contains(guid)) {
- context.guidsToProcess.add(guid);
+ if (!context.guidsToProcessContains(guid)) {
+ context.guidsToProcessAdd(guid);
}
} else if (currentDirection == TraversalDirection.OUTWARD && direction == TraversalDirection.INWARD) {
context.guidDirection.put(guid, direction);
@@ -298,14 +299,14 @@ public class ExportService {
// the entity should be reprocessed to get inward entities
context.guidsProcessed.remove(guid);
- if (!context.guidsToProcess.contains(guid)) {
- context.guidsToProcess.add(guid);
+ if (!context.guidsToProcessContains(guid)) {
+ context.guidsToProcessAdd(guid);
}
}
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== getConnectedEntityGuids({}): found {} guids; guidsToProcess {}", entity.getGuid(), guids.size(), context.guidsToProcess.size());
+ LOG.debug("<== getConnectedEntityGuids({}): found {} guids; guidsToProcess {}", entity.getGuid(), guids.size(), context.guidsToProcessSize());
}
}
}
@@ -323,7 +324,7 @@ public class ExportService {
private void getEntityGuidsForFullFetch(AtlasEntity entity, ExportContext context) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> getEntityGuidsForFullFetch({}): guidsToProcess {}", AtlasTypeUtil.getAtlasObjectId(entity), context.guidsToProcess.size());
+ LOG.debug("==> getEntityGuidsForFullFetch({}): guidsToProcess {}", AtlasTypeUtil.getAtlasObjectId(entity), context.guidsToProcessSize());
}
String query = this.gremlinQueryProvider.getQuery(AtlasGremlinQuery.EXPORT_BY_GUID_FULL);
@@ -339,8 +340,8 @@ public class ExportService {
for (String guid : result) {
if (!context.guidsProcessed.contains(guid)) {
- if (!context.guidsToProcess.contains(guid)) {
- context.guidsToProcess.add(guid);
+ if (!context.guidsToProcessContains(guid)) {
+ context.guidsToProcessAdd(guid);
}
context.guidDirection.put(guid, TraversalDirection.BOTH);
@@ -348,7 +349,7 @@ public class ExportService {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== getEntityGuidsForFullFetch({}): found {} guids; guidsToProcess {}", entity.getGuid(), result.size(), context.guidsToProcess.size());
+ LOG.debug("<== getEntityGuidsForFullFetch({}): found {} guids; guidsToProcess {}", entity.getGuid(), result.size(), context.guidsToProcessSize());
}
}
@@ -434,7 +435,8 @@ public class ExportService {
private class ExportContext {
final Set<String> guidsProcessed = new HashSet<>();
- final List<String> guidsToProcess = new ArrayList<>();
+ private final List<String> guidsToProcessList = new ArrayList<>();
+ private final Set<String> guidsToProcessSet = new HashSet<>();
final Map<String, TraversalDirection> guidDirection = new HashMap<>();
final AtlasExportResult result;
final ZipSink sink;
@@ -477,5 +479,35 @@ public class ExportService {
return matchType;
}
+
+ public void clear() {
+ guidsToProcessList.clear();
+ guidsToProcessSet.clear();
+ guidsProcessed.clear();
+ guidDirection.clear();
+ }
+
+ public boolean guidsToProcessIsEmpty() {
+ return this.guidsToProcessList.isEmpty();
+ }
+
+ public String guidsToProcessRemove(int i) {
+ String s = this.guidsToProcessList.remove(i);
+ guidsToProcessSet.remove(s);
+ return s;
+ }
+
+ public int guidsToProcessSize() {
+ return this.guidsToProcessList.size();
+ }
+
+ public boolean guidsToProcessContains(String guid) {
+ return guidsToProcessSet.contains(guid);
+ }
+
+ public void guidsToProcessAdd(String guid) {
+ this.guidsToProcessList.add(guid);
+ guidsToProcessSet.add(guid);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/5b59acd7/webapp/src/main/java/org/apache/atlas/web/resources/ZipSink.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/ZipSink.java b/webapp/src/main/java/org/apache/atlas/web/resources/ZipSink.java
index 2e4cb01..37d9eb5 100644
--- a/webapp/src/main/java/org/apache/atlas/web/resources/ZipSink.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/ZipSink.java
@@ -18,18 +18,16 @@
package org.apache.atlas.web.resources;
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
-import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -37,15 +35,9 @@ public class ZipSink {
private static final Logger LOG = LoggerFactory.getLogger(ZipSink.class);
private ZipOutputStream zipOutputStream;
- private ByteArrayOutputStream byteArrayOutputStream;
-
- public ZipSink() {
- init();
- }
- private void init() {
- byteArrayOutputStream = new ByteArrayOutputStream();
- zipOutputStream = new ZipOutputStream(byteArrayOutputStream);
+ public ZipSink(OutputStream outputStream) {
+ zipOutputStream = new ZipOutputStream(outputStream);
}
public void add(AtlasEntity entity) throws AtlasBaseException {
@@ -68,10 +60,6 @@ public class ZipSink {
saveToZip(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME, jsonData);
}
- public void writeTo(OutputStream stream) throws IOException {
- byteArrayOutputStream.writeTo(stream);
- }
-
public void close() {
try {
if(zipOutputStream != null) {