You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ni...@apache.org on 2020/05/22 13:22:04 UTC
[atlas] branch master updated: ATLAS-3802 : BasicSearch: filter,
attribute with NEQ operator doesn't match attribute having null
values.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 7718c5e ATLAS-3802 : BasicSearch: filter, attribute with NEQ operator doesn't match attribute having null values.
7718c5e is described below
commit 7718c5eb4955818befefcd7d9768f1cdb202cff2
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Thu May 21 17:31:36 2020 +0530
ATLAS-3802 : BasicSearch: filter, attribute with NEQ operator doesn't match attribute having null values.
Change-Id: I19141c1a0917aa2d761cc6429857d4f56c272094
Signed-off-by: nixonrodrigues <ni...@apache.org>
---
.../org/apache/atlas/util/SearchPredicateUtil.java | 102 +++++++++++++++------
.../test/java/org/apache/atlas/BasicTestSetup.java | 24 +++++
.../discovery/BasicSearchClassificationTest.java | 14 +--
.../atlas/discovery/EntitySearchProcessorTest.java | 71 ++++++++++++++
4 files changed, 172 insertions(+), 39 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java b/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
index 4408cd4..5069d78 100644
--- a/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
+++ b/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
@@ -279,7 +279,7 @@ public class SearchPredicateUtil {
public Predicate generatePredicate(final String attrName, final Object attrVal, final Class attrClass) {
final Predicate ret;
- if (attrName == null || attrClass == null || attrVal == null) {
+ if (attrName == null || attrClass == null) {
ret = ALWAYS_FALSE;
} else if (Boolean.class.isAssignableFrom(attrClass)) {
ret = BooleanPredicate.getNEQPredicate(attrName, attrClass, (Boolean)attrVal);
@@ -585,16 +585,16 @@ public class SearchPredicateUtil {
public Predicate generatePredicate(final String attrName, final Object attrVal, final Class attrClass) {
final Predicate ret;
- if (attrName == null || attrClass == null || attrVal == null) {
+ if (attrName == null || attrClass == null) {
ret = ALWAYS_FALSE;
} else if (String.class.isAssignableFrom(attrClass)) {
ret = StringPredicate.getNotContainsPredicate(attrName, attrClass, (String) attrVal);
} else if (Collection.class.isAssignableFrom(attrClass)) {
// Check if the provided value is present in the list of stored values
- ret = new VertexAttributePredicate(attrName, attrClass) {
+ ret = new VertexAttributePredicate(attrName, attrClass,true) {
@Override
protected boolean compareValue(final Object vertexAttrVal) {
- return !((Collection) vertexAttrVal).contains(attrVal);
+ return vertexAttrVal == null || !((Collection) vertexAttrVal).contains(attrVal);
}
};
} else {
@@ -817,6 +817,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ BooleanPredicate(String attrName, Class attrClass, Boolean value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, Boolean value) {
return new SearchPredicateUtil.BooleanPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -826,9 +831,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, Boolean value) {
- return new SearchPredicateUtil.BooleanPredicate(attrName, attrClass, value) {
+ return new SearchPredicateUtil.BooleanPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((Boolean) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((Boolean) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -843,6 +848,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ ShortPredicate(String attrName, Class attrClass, Short value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, Short value) {
return new ShortPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -852,9 +862,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, Short value) {
- return new ShortPredicate(attrName, attrClass, value) {
+ return new ShortPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((Short) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((Short) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -901,6 +911,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ IntegerPredicate(String attrName, Class attrClass, Integer value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, Integer value) {
return new IntegerPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -910,9 +925,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, Integer value) {
- return new IntegerPredicate(attrName, attrClass, value) {
+ return new IntegerPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((Integer) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((Integer) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -959,6 +974,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ LongPredicate(String attrName, Class attrClass, Long value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, Long value) {
return new LongPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -968,9 +988,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, Long value) {
- return new LongPredicate(attrName, attrClass, value) {
+ return new LongPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((Long) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((Long) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -1017,6 +1037,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ FloatPredicate(String attrName, Class attrClass, Float value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, Float value) {
return new FloatPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -1026,9 +1051,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, Float value) {
- return new FloatPredicate(attrName, attrClass, value) {
+ return new FloatPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((Float) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((Float) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -1075,6 +1100,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ DoublePredicate(String attrName, Class attrClass, Double value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, Double value) {
return new DoublePredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -1084,9 +1114,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, Double value) {
- return new DoublePredicate(attrName, attrClass, value) {
+ return new DoublePredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((Double) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((Double) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -1133,6 +1163,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ BytePredicate(String attrName, Class attrClass, Byte value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, Byte value) {
return new BytePredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -1142,9 +1177,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, Byte value) {
- return new BytePredicate(attrName, attrClass, value) {
+ return new BytePredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((Byte) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((Byte) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -1191,6 +1226,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ BigIntegerPredicate(String attrName, Class attrClass, BigInteger value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, BigInteger value) {
return new BigIntegerPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -1200,9 +1240,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, BigInteger value) {
- return new BigIntegerPredicate(attrName, attrClass, value) {
+ return new BigIntegerPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((BigInteger) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((BigInteger) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -1249,6 +1289,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ BigDecimalPredicate(String attrName, Class attrClass, BigDecimal value, boolean isNullValid) {
+ super(attrName, attrClass, true);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, BigDecimal value) {
return new BigDecimalPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -1258,9 +1303,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, BigDecimal value) {
- return new BigDecimalPredicate(attrName, attrClass, value) {
+ return new BigDecimalPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((BigDecimal) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((BigDecimal) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -1307,6 +1352,11 @@ public class SearchPredicateUtil {
this.value = value;
}
+ StringPredicate(String attrName, Class attrClass, String value, boolean isNullValid) {
+ super(attrName, attrClass, isNullValid);
+ this.value = value;
+ }
+
static VertexAttributePredicate getEQPredicate(String attrName, Class attrClass, String value) {
return new StringPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {
@@ -1316,9 +1366,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNEQPredicate(String attrName, Class attrClass, String value) {
- return new StringPredicate(attrName, attrClass, value) {
+ return new StringPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return ((String) vertexAttrVal).compareTo(value) != 0;
+ return vertexAttrVal == null || ((String) vertexAttrVal).compareTo(value) != 0;
}
};
}
@@ -1364,9 +1414,9 @@ public class SearchPredicateUtil {
}
static VertexAttributePredicate getNotContainsPredicate(String attrName, Class attrClass, String value) {
- return new StringPredicate(attrName, attrClass, value) {
+ return new StringPredicate(attrName, attrClass, value, true) {
protected boolean compareValue(Object vertexAttrVal) {
- return !((String) vertexAttrVal).contains(value);
+ return vertexAttrVal == null || !((String) vertexAttrVal).contains(value);
}
};
}
diff --git a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
index 7508f78..8b98b39 100644
--- a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
+++ b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
@@ -19,6 +19,7 @@ package org.apache.atlas;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
@@ -448,4 +449,27 @@ public abstract class BasicTestSetup {
return datasetSubType;
}
+
+ public void createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException {
+ AtlasEntity entity = new AtlasEntity(type);
+ entity.setAttribute("name", name);
+ entity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
+ entity.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList()));
+ entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false);
+ }
+
+ public SearchParameters.FilterCriteria getSingleFilterCondition(String attName, SearchParameters.Operator op, String attrValue) {
+ SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria();
+ filterCriteria.setCondition(SearchParameters.FilterCriteria.Condition.AND);
+ List<SearchParameters.FilterCriteria> criteria = new ArrayList<>();
+ SearchParameters.FilterCriteria f1 = new SearchParameters.FilterCriteria();
+ f1.setAttributeName(attName);
+ f1.setOperator(op);
+ String time = String.valueOf(System.currentTimeMillis());
+ f1.setAttributeValue(attrValue);
+ criteria.add(f1);
+ filterCriteria.setCriterion(criteria);
+ return filterCriteria;
+ }
+
}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java b/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java
index 95f782a..9b16e91 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java
@@ -225,17 +225,5 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
return entityHeaders.size();
}
- private FilterCriteria getSingleFilterCondition(String attName, Operator op, String attrValue) {
- FilterCriteria filterCriteria = new FilterCriteria();
- filterCriteria.setCondition(FilterCriteria.Condition.AND);
- List<FilterCriteria> criteria = new ArrayList<>();
- FilterCriteria f1 = new FilterCriteria();
- f1.setAttributeName(attName);
- f1.setOperator(op);
- String time = String.valueOf(System.currentTimeMillis());
- f1.setAttributeValue(attrValue);
- criteria.add(f1);
- filterCriteria.setCriterion(criteria);
- return filterCriteria;
- }
+
}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
index 1951c39..43f11d1 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
@@ -23,6 +23,7 @@ import org.apache.atlas.SortOrder;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
@@ -32,6 +33,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
@@ -56,6 +58,8 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
setupTestData();
}
+ @Inject
+ public GraphBackedSearchIndexer indexer;
@Test
public void searchTablesByClassification() throws AtlasBaseException {
@@ -126,4 +130,71 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
new EntitySearchProcessor(context);
}
+ @Test
+ public void searchWithNEQ_stringAttr() throws AtlasBaseException {
+ String expectedEntityName = "hive_Table_Null_tableType";
+ createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.NEQ, "Managed");
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ assertEquals(vertices.size(), 3);
+
+ List<String> nameList = new ArrayList<>();
+ for (AtlasVertex vertex : vertices) {
+ nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
+ }
+
+ assertTrue(nameList.contains(expectedEntityName));
+ }
+
+ @Test(dependsOnMethods = "searchWithNEQ_stringAttr")
+ public void searchWithNEQ_pipeSeperatedAttr() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("__classificationNames", SearchParameters.Operator.NEQ, METRIC_CLASSIFICATION);
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ assertEquals(vertices.size(), 7);
+
+ List<String> nameList = new ArrayList<>();
+ for (AtlasVertex vertex : vertices) {
+ nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
+ }
+
+ assertTrue(nameList.contains("hive_Table_Null_tableType"));
+ }
+
+ @Test(dependsOnMethods = "searchWithNEQ_stringAttr")
+ public void searchWithNEQ_doubleAttr() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("retention", SearchParameters.Operator.NEQ, "5");
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ assertEquals(vertices.size(), 1);
+
+ List<String> nameList = new ArrayList<>();
+ for (AtlasVertex vertex : vertices) {
+ nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
+ }
+
+ assertTrue(nameList.contains("hive_Table_Null_tableType"));
+ }
}