You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/09/15 19:08:42 UTC
[atlas] branch master updated: ATLAS-3367: updated search-result to
include approximate-total-count if available
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 5ab0947 ATLAS-3367: updated search-result to include approximate-total-count if available
5ab0947 is described below
commit 5ab09475e097af72f70694518bb65bb6ac0d868d
Author: Bolke de Bruin <bo...@xs4all.nl>
AuthorDate: Mon Aug 12 14:16:44 2019 +0200
ATLAS-3367: updated search-result to include approximate-total-count if available
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
.../org/apache/atlas/model/discovery/AtlasSearchResult.java | 6 ++++++
.../apache/atlas/discovery/ClassificationSearchProcessor.java | 5 +++++
.../org/apache/atlas/discovery/EntityDiscoveryService.java | 11 +++++++++--
.../org/apache/atlas/discovery/EntitySearchProcessor.java | 5 +++++
.../org/apache/atlas/discovery/FreeTextSearchProcessor.java | 5 +++++
.../org/apache/atlas/discovery/FullTextSearchProcessor.java | 5 +++++
.../main/java/org/apache/atlas/discovery/SearchProcessor.java | 1 +
.../java/org/apache/atlas/discovery/TermSearchProcessor.java | 5 +++++
8 files changed, 41 insertions(+), 2 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
index d274cc0..e1c550e 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
@@ -52,6 +52,7 @@ public class AtlasSearchResult implements Serializable {
private AttributeSearchResult attributes;
private List<AtlasFullTextResult> fullTextResult;
private Map<String, AtlasEntityHeader> referredEntities;
+ private long approximateCount = -1;
public AtlasSearchResult() {}
@@ -126,6 +127,10 @@ public class AtlasSearchResult implements Serializable {
this.referredEntities = referredEntities;
}
+ public long getApproximateCount() { return approximateCount; }
+
+ public void setApproximateCount(long approximateCount) { this.approximateCount = approximateCount; }
+
@Override
public int hashCode() { return Objects.hash(queryType, searchParameters, queryText, type, classification, entities, attributes, fullTextResult, referredEntities); }
@@ -184,6 +189,7 @@ public class AtlasSearchResult implements Serializable {
", attributes=" + attributes +
", fullTextResult=" + fullTextResult +
", referredEntities=" + referredEntities +
+ ", approximateCount=" + approximateCount +
'}';
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 479ddfd..b457984 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -347,4 +347,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.debug("<== ClassificationSearchProcessor.filter(): ret.size()={}", entityVertices.size());
}
}
+
+ @Override
+ public long getResultCount() {
+ return (indexQuery != null) ? indexQuery.vertexTotals() : -1;
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index f667aa3..23fd7e9 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -180,6 +180,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
LOG.debug("Executing Full text query: {}", fullTextQuery);
}
ret.setFullTextResult(getIndexQueryResults(idxQuery, params, excludeDeletedEntities));
+ ret.setApproximateCount(idxQuery.vertexTotals());
scrubSearchResults(ret);
@@ -285,7 +286,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
int resultIdx = 0;
for (int indexQueryOffset = 0; ; indexQueryOffset += getMaxResultSetSize()) {
- final Iterator<Result<?, ?>> qryResult = graph.indexQuery(Constants.FULLTEXT_INDEX, idxQuery, indexQueryOffset).vertices();
+ final AtlasIndexQuery qry = graph.indexQuery(Constants.FULLTEXT_INDEX, idxQuery, indexQueryOffset);
+ final Iterator<Result<?, ?>> qryResult = qry.vertices();
if (LOG.isDebugEnabled()) {
LOG.debug("indexQuery: query=" + idxQuery + "; offset=" + indexQueryOffset);
@@ -344,6 +346,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
}
}
+ if (ret.getApproximateCount() < 0) {
+ ret.setApproximateCount(qry.vertexTotals());
+ }
+
if (ret.getEntities() != null && ret.getEntities().size() == resultSize) {
break;
}
@@ -393,7 +399,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
if (firstElement instanceof AtlasVertex) {
for (Object element : queryResult) {
if (element instanceof AtlasVertex) {
-
ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex) element));
} else {
LOG.warn("searchUsingBasicQuery({}): expected an AtlasVertex; found unexpected entry in result {}", basicQuery, element);
@@ -465,6 +470,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
try {
List<AtlasVertex> resultList = searchContext.getSearchProcessor().execute();
+ ret.setApproximateCount(searchContext.getSearchProcessor().getResultCount());
+
// By default any attribute that shows up in the search parameter should be sent back in the response
// If additional values are requested then the entityAttributes will be a superset of the all search attributes
// and the explicitly requested attribute(s)
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index f7d8f08..2e07814 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -311,4 +311,9 @@ public class EntitySearchProcessor extends SearchProcessor {
LOG.debug("<== EntitySearchProcessor.filter(): ret.size()={}", entityVertices.size());
}
}
+
+ @Override
+ public long getResultCount() {
+ return (indexQuery != null) ? indexQuery.vertexTotals() : -1;
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
index 1dd1afa..d628f13 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
@@ -174,4 +174,9 @@ public class FreeTextSearchProcessor extends SearchProcessor {
return ret;
}
+
+ @Override
+ public long getResultCount() {
+ return indexQuery.vertexTotals();
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 0ffd61c..152ade8 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -166,4 +166,9 @@ public class FullTextSearchProcessor extends SearchProcessor {
return ret;
}
+
+ @Override
+ public long getResultCount() {
+ return indexQuery.vertexTotals();
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index f7847ed..a12ce3f 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -128,6 +128,7 @@ public abstract class SearchProcessor {
}
public abstract List<AtlasVertex> execute();
+ public abstract long getResultCount();
protected int collectResultVertices(final List<AtlasVertex> ret, final int startIdx, final int limit, int resultIdx, final List<AtlasVertex> entityVertices) {
for (AtlasVertex entityVertex : entityVertices) {
diff --git a/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java
index c253d54..6c48a82 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java
@@ -110,4 +110,9 @@ public class TermSearchProcessor extends SearchProcessor {
LOG.debug("<== TermSearchProcessor.filter(): ret.size()={}", entityVertices.size());
}
}
+
+ @Override
+ public long getResultCount() {
+ return -1;
+ }
}