You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/09/22 22:16:39 UTC
atlas git commit: ATLAS-2092: Create edgeLabel while updating the
index
Repository: atlas
Updated Branches:
refs/heads/master 20f2fba7e -> d6d349e5e
ATLAS-2092: Create edgeLabel while updating the index
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/d6d349e5
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/d6d349e5
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/d6d349e5
Branch: refs/heads/master
Commit: d6d349e5e11d5dad4c888dc182fbf1daddf80761
Parents: 20f2fba
Author: apoorvnaik <ap...@apache.org>
Authored: Wed Sep 20 14:35:14 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Sep 22 15:16:25 2017 -0700
----------------------------------------------------------------------
.../repository/graphdb/AtlasEdgeLabel.java | 28 +++++
.../graphdb/AtlasGraphManagement.java | 12 ++
.../graphdb/titan0/GraphDbObjectFactory.java | 12 ++
.../graphdb/titan0/Titan0EdgeLabel.java | 64 +++++++++++
.../graphdb/titan0/Titan0GraphManagement.java | 14 +++
.../graphdb/titan1/GraphDbObjectFactory.java | 12 ++
.../graphdb/titan1/Titan1EdgeLabel.java | 59 ++++++++++
.../graphdb/titan1/Titan1GraphManagement.java | 14 +++
.../graph/GraphBackedSearchIndexer.java | 41 ++++++-
.../test/java/org/apache/atlas/TestUtils.java | 8 +-
.../discovery/DataSetLineageServiceTest.java | 4 +-
.../atlas/lineage/EntityLineageServiceTest.java | 3 +-
.../audit/AuditRepositoryTestBase.java | 3 +-
.../GraphBackedMetadataRepositoryTest.java | 4 +-
.../graph/GraphBackedSearchIndexerTest.java | 3 +-
.../store/graph/v1/AtlasEntityStoreV1Test.java | 7 +-
.../service/DefaultMetadataServiceTest.java | 111 ++++++++++++++++++-
.../apache/atlas/query/QueryTestsUtils.scala | 5 +-
18 files changed, 376 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeLabel.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeLabel.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeLabel.java
new file mode 100644
index 0000000..3a210a2
--- /dev/null
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeLabel.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb;
+
+/**
+ * Represent an edge label.
+ *
+ */
+public interface AtlasEdgeLabel {
+
+ String getName();
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
index 0db46d4..620ba10 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
@@ -63,6 +63,12 @@ public interface AtlasGraphManagement {
AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, AtlasCardinality cardinality);
/**
+ *
+ * @param label edge label to be created
+ */
+ AtlasEdgeLabel makeEdgeLabel(String label);
+
+ /**
* @param propertyKey
*
*/
@@ -75,6 +81,12 @@ public interface AtlasGraphManagement {
AtlasPropertyKey getPropertyKey(String propertyName);
/**
+ * @param label
+ * @return
+ */
+ AtlasEdgeLabel getEdgeLabel(String label);
+
+ /**
* Creates a composite index for the graph.
*
* @param propertyName
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
index 0e8519e..73097f2 100644
--- a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.titan0;
+import com.thinkaurelius.titan.core.EdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
@@ -88,6 +89,17 @@ public final class GraphDbObjectFactory {
}
/**
+ * @param label The label.
+ *
+ */
+ public static Titan0EdgeLabel createEdgeLabel(EdgeLabel label) {
+ if (label == null) {
+ return null;
+ }
+ return new Titan0EdgeLabel(label);
+ }
+
+ /**
* @param index The gremlin index.
* @return
*/
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0EdgeLabel.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0EdgeLabel.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0EdgeLabel.java
new file mode 100644
index 0000000..934e255
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0EdgeLabel.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan0;
+
+import com.thinkaurelius.titan.core.EdgeLabel;
+import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
+
+/**
+ * Titan 0.5.4 implementaiton of AtlasEdgeLabel.
+ */
+public class Titan0EdgeLabel implements AtlasEdgeLabel {
+ private final EdgeLabel wrappedEdgeLabel;
+
+ public Titan0EdgeLabel(EdgeLabel toWrap) {
+ wrappedEdgeLabel = toWrap;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.atlas.repository.graphdb.AtlasEdgeLabel#getName()
+ */
+ @Override
+ public String getName() {
+ return wrappedEdgeLabel.getName();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Titan0EdgeLabel)) {
+ return false;
+ }
+ Titan0EdgeLabel otherLabel = (Titan0EdgeLabel) other;
+ return wrappedEdgeLabel.equals(otherLabel.wrappedEdgeLabel);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ result = 37 * result + wrappedEdgeLabel.hashCode();
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return wrappedEdgeLabel.getName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java
index ec4d6c4..5b9c933 100644
--- a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphManagement.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.titan0;
import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.PropertyKeyMaker;
@@ -27,6 +28,7 @@ import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
+import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
@@ -111,6 +113,13 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
}
@Override
+ public AtlasEdgeLabel makeEdgeLabel(String label) {
+ EdgeLabel edgeLabel = management.makeEdgeLabel(label).make();
+
+ return GraphDbObjectFactory.createEdgeLabel(edgeLabel);
+ }
+
+ @Override
public void deletePropertyKey(String propertyKey) {
PropertyKey titanPropertyKey = management.getPropertyKey(propertyKey);
@@ -132,6 +141,11 @@ public class Titan0GraphManagement implements AtlasGraphManagement {
}
@Override
+ public AtlasEdgeLabel getEdgeLabel(String label) {
+ return GraphDbObjectFactory.createEdgeLabel(management.getEdgeLabel(label));
+ }
+
+ @Override
public void createExactMatchIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java
----------------------------------------------------------------------
diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java
index c27bc35..7928a84 100644
--- a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java
+++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/GraphDbObjectFactory.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.titan1;
+import com.thinkaurelius.titan.core.EdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.titan1.query.Titan1GraphQuery;
@@ -89,6 +90,17 @@ public final class GraphDbObjectFactory {
}
/**
+ * @param label The Gremlin propertyKey.
+ *
+ */
+ public static Titan1EdgeLabel createEdgeLabel(EdgeLabel label) {
+ if (label == null) {
+ return null;
+ }
+ return new Titan1EdgeLabel(label);
+ }
+
+ /**
* @param index The gremlin index.
* @return
*/
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1EdgeLabel.java
----------------------------------------------------------------------
diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1EdgeLabel.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1EdgeLabel.java
new file mode 100644
index 0000000..fdcc33d
--- /dev/null
+++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1EdgeLabel.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan1;
+
+import com.thinkaurelius.titan.core.EdgeLabel;
+import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
+
+/**
+ *
+ */
+public class Titan1EdgeLabel implements AtlasEdgeLabel {
+
+ private final EdgeLabel wrapped;
+
+ public Titan1EdgeLabel(EdgeLabel toWrap) {
+ wrapped = toWrap;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.repository.graphdb.AtlasPropertyKey#getName()
+ */
+ @Override
+ public String getName() {
+ return wrapped.name();
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ result = 37*result + wrapped.hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Titan1EdgeLabel)) {
+ return false;
+ }
+ Titan1EdgeLabel otherKey = (Titan1EdgeLabel)other;
+ return otherKey.wrapped.equals(wrapped);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphManagement.java
----------------------------------------------------------------------
diff --git a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphManagement.java b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphManagement.java
index 61dc298..a5d3815 100644
--- a/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphManagement.java
+++ b/graphdb/titan1/src/main/java/org/apache/atlas/repository/graphdb/titan1/Titan1GraphManagement.java
@@ -19,6 +19,7 @@ package org.apache.atlas.repository.graphdb.titan1;
import com.google.common.base.Preconditions;
import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.PropertyKeyMaker;
@@ -26,6 +27,7 @@ import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.thinkaurelius.titan.graphdb.internal.Token;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
+import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
@@ -135,6 +137,13 @@ public class Titan1GraphManagement implements AtlasGraphManagement {
}
@Override
+ public AtlasEdgeLabel makeEdgeLabel(String label) {
+ EdgeLabel edgeLabel = management.makeEdgeLabel(label).make();
+
+ return GraphDbObjectFactory.createEdgeLabel(edgeLabel);
+ }
+
+ @Override
public void deletePropertyKey(String propertyKey) {
PropertyKey titanPropertyKey = management.getPropertyKey(propertyKey);
@@ -155,6 +164,11 @@ public class Titan1GraphManagement implements AtlasGraphManagement {
return GraphDbObjectFactory.createPropertyKey(management.getPropertyKey(propertyName));
}
+ @Override
+ public AtlasEdgeLabel getEdgeLabel(String label) {
+ return GraphDbObjectFactory.createEdgeLabel(management.getEdgeLabel(label));
+ }
+
public void createExactMatchVertexIndex(String propertyName, boolean enforceUniqueness,
List<AtlasPropertyKey> propertyKeys) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index be6a0a0..8aa2237 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -190,8 +190,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
//Indexes for graph backed type system store
createTypeStoreIndexes(management);
-
-
+
commit(management);
LOG.info("Index creation for global keys complete.");
} catch (Throwable t) {
@@ -319,8 +318,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
try {
AtlasType atlasType = typeRegistry.getType(attribTypeName);
- if (isMapType || isArrayType || isClassificationType(atlasType) || isEntityType(atlasType)) {
+ if (isMapType || isClassificationType(atlasType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
+ } if (isArrayType) {
+ createLabelIfNeeded(management, propertyName, attribTypeName);
+ } if (isEntityType(atlasType)) {
+ createEdgeLabel(management, propertyName);
} else if (isBuiltInType) {
createIndexes(management, propertyName, getPrimitiveClass(attribTypeName), isUnique, cardinality, false, isIndexable);
} else if (isEnumType(atlasType)) {
@@ -334,6 +337,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+ private void createLabelIfNeeded(final AtlasGraphManagement management, final String propertyName, final String attribTypeName) {
+ // If any of the referenced typename is of type Entity or Struct then the edge label needs to be created
+ for (String typeName : AtlasTypeUtil.getReferencedTypeNames(attribTypeName)) {
+ if (typeRegistry.getEntityDefByName(typeName) != null || typeRegistry.getStructDefByName(typeName) != null) {
+ // Create the edge label upfront to avoid running into concurrent call issue (ATLAS-2092)
+ createEdgeLabel(management, propertyName);
+ }
+ }
+ }
+
private boolean isEntityType(AtlasType type) {
return type instanceof AtlasEntityType;
}
@@ -442,6 +455,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
break;
case ARRAY:
+ createLabelIfNeeded(management, propertyName, field.dataType().getName());
+ break;
case MAP:
// todo - how do we overcome this limitation?
// IGNORE: Can only index single-valued property keys on vertices in Mixed Index
@@ -457,9 +472,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
break;
case CLASS:
- // this is only A reference, index the attribute for edge
- // Commenting this out since we do not need an index for edge here
- //createEdgeMixedIndex(propertyName);
+ createEdgeLabel(management, propertyName);
break;
default:
@@ -467,6 +480,22 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+ private void createEdgeLabel(final AtlasGraphManagement management, final String propertyName) {
+ // Create the edge label upfront to avoid running into concurrent call issue (ATLAS-2092)
+ // ATLAS-2092 addresses this problem by creating the edge label upfront while type creation
+ // which resolves the race condition during the entity creation
+
+ String label = Constants.INTERNAL_PROPERTY_KEY_PREFIX + propertyName;
+
+ org.apache.atlas.repository.graphdb.AtlasEdgeLabel edgeLabel = management.getEdgeLabel(label);
+
+ if (edgeLabel == null) {
+ management.makeEdgeLabel(label);
+
+ LOG.info("Created edge label {} ", label);
+ }
+ }
+
private Class getPrimitiveClass(IDataType dataType) {
if (dataType == DataTypes.STRING_TYPE) {
return String.class;
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
index 1e439f7..4891279 100755
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -481,7 +481,13 @@ public final class TestUtils {
}
public static final String randomString() {
- return RandomStringUtils.randomAlphanumeric(10);
+ return randomString(10);
+ }
+
+ public static final String randomString(int count) {
+ final String prefix = "r";
+
+ return prefix + RandomStringUtils.randomAlphanumeric(count - prefix.length()); // ensure that the string starts with a letter
}
public static Referenceable createDBEntity() {
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
index c700a61..3db58fe 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.BaseRepositoryTest;
import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtils;
import org.apache.atlas.model.legacy.EntityResult;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
@@ -31,7 +32,6 @@ import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.commons.collections.ArrayStack;
-import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -488,7 +488,7 @@ public class DataSetLineageServiceTest extends BaseRepositoryTest {
}
private String random() {
- return RandomStringUtils.randomAlphanumeric(5);
+ return TestUtils.randomString(5);
}
private String getEntityId(String typeName, String attributeName, String attributeValue) throws Exception {
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java
index 8a88f32..202f20c 100644
--- a/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/lineage/EntityLineageServiceTest.java
@@ -34,7 +34,6 @@ import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.commons.collections.ArrayStack;
-import org.apache.commons.lang.RandomStringUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -347,7 +346,7 @@ public class EntityLineageServiceTest extends BaseRepositoryTest {
}
private String random() {
- return RandomStringUtils.randomAlphanumeric(5);
+ return TestUtils.randomString(5);
}
private String getEntityId(String typeName, String attributeName, String attributeValue) throws Exception {
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
index 5bde605..551e6ab 100644
--- a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
@@ -21,7 +21,6 @@ package org.apache.atlas.repository.audit;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.TestUtils;
import org.apache.atlas.typesystem.Referenceable;
-import org.apache.commons.lang.RandomStringUtils;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@@ -35,7 +34,7 @@ public class AuditRepositoryTestBase {
protected EntityAuditRepository eventRepository;
private String rand() {
- return RandomStringUtils.randomAlphanumeric(10);
+ return TestUtils.randomString(10);
}
@BeforeTest
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
index f372891..9167922 100755
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
@@ -885,11 +885,11 @@ public class GraphBackedMetadataRepositoryTest {
}
private String randomUTF() {
- return RandomStringUtils.random(10);
+ RandomStringUtils.random(10);
}
private String randomString() {
- return RandomStringUtils.randomAlphanumeric(10);
+ return TestUtils.randomString(10);
}
@Test
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
index ef5d469..feffabf 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
@@ -34,7 +34,6 @@ import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.RandomStringUtils;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@@ -136,7 +135,7 @@ public class GraphBackedSearchIndexerTest {
try {
TypeSystem typeSystem = TypeSystem.getInstance();
- String enumName = "randomEnum" + RandomStringUtils.randomAlphanumeric(10);
+ String enumName = "randomEnum" + TestUtils.randomString(10);
EnumType managedType = typeSystem.defineEnumType(enumName, new EnumValue("randomEnumValue", 0));
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
index 8d04094..ef8bd2f 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
@@ -55,7 +55,6 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.RandomStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -629,7 +628,7 @@ public class AtlasEntityStoreV1Test {
//TODO : Failing in typedef creation
public void testSpecialCharacters() throws Exception {
//Verify that type can be created with reserved characters in typename, attribute name
- final String typeName = "test_type_"+ RandomStringUtils.randomAlphanumeric(10);
+ final String typeName = TestUtils.randomString(10);
String strAttrName = randomStrWithReservedChars();
String arrayAttrName = randomStrWithReservedChars();
String mapAttrName = randomStrWithReservedChars();
@@ -685,7 +684,7 @@ public class AtlasEntityStoreV1Test {
init();
AtlasEntity dbEntity = new AtlasEntity(TestUtilsV2.DATABASE_TYPE);
- dbEntity.setAttribute("name", RandomStringUtils.randomAlphanumeric(10));
+ dbEntity.setAttribute("name", TestUtils.randomString(10));
dbEntity.setAttribute("description", "us db");
dbEntity.setAttribute("isReplicated", false);
dbEntity.setAttribute("created", "09081988");
@@ -731,7 +730,7 @@ public class AtlasEntityStoreV1Test {
// create a new table type
AtlasEntity tblEntity = new AtlasEntity(TABLE_TYPE);
- tblEntity.setAttribute("name", RandomStringUtils.randomAlphanumeric(10));
+ tblEntity.setAttribute("name", TestUtils.randomString(10));
tblEntity.setAttribute("type", "type");
tblEntity.setAttribute("tableType", "MANAGED");
tblEntity.setAttribute("database", AtlasTypeUtil.getAtlasObjectId(updatedDbEntity));
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
index d8d8d91..e93f08d 100644
--- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
@@ -23,9 +23,10 @@ import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.CreateUpdateEntitiesResult;
import org.apache.atlas.EntityAuditEvent;
-import org.apache.atlas.TestModules;
import org.apache.atlas.RequestContext;
+import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
@@ -34,10 +35,15 @@ import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.model.legacy.EntityResult;
import org.apache.atlas.query.QueryParams;
+import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
import org.apache.atlas.repository.audit.HBaseTestUtils;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasTypeUtil;
@@ -63,7 +69,6 @@ import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
-import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -80,6 +85,11 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import static org.apache.atlas.TestUtils.*;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
@@ -164,7 +174,7 @@ public class DefaultMetadataServiceTest {
@Test(expectedExceptions = TypeNotFoundException.class)
public void testCreateEntityWithUnknownDatatype() throws Exception {
Referenceable entity = new Referenceable("Unknown datatype");
- String dbName = RandomStringUtils.randomAlphanumeric(10);
+ String dbName = TestUtils.randomString(10);
entity.set(NAME, dbName);
entity.set("description", "us db");
TestUtils.createInstance(metadataService, entity);
@@ -197,7 +207,7 @@ public class DefaultMetadataServiceTest {
String arrayAttrName = randomStrWithReservedChars();
String mapAttrName = randomStrWithReservedChars();
HierarchicalTypeDefinition<ClassType> typeDefinition =
- createClassTypeDef("test_type_"+ RandomStringUtils.randomAlphanumeric(10), ImmutableSet.<String>of(),
+ createClassTypeDef("test_type_"+ TestUtils.randomString(10), ImmutableSet.<String>of(),
createOptionalAttrDef(strAttrName, DataTypes.STRING_TYPE),
new AttributeDefinition(arrayAttrName, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
@@ -1112,7 +1122,7 @@ public class DefaultMetadataServiceTest {
@Test
public void testTypeUpdateFailureShouldRollBack() throws AtlasException, JSONException {
- String typeName = "test_type_"+ RandomStringUtils.randomAlphanumeric(10);
+ String typeName = TestUtils.randomString(10);
HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef(
typeName, ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("test_type_attribute", DataTypes.STRING_TYPE));
@@ -1227,6 +1237,97 @@ public class DefaultMetadataServiceTest {
}
}
+ @Test
+ // ATLAS-2092: Concurrent edge label creation leads to inconsistency
+ // This test tries entity creation in parallel and ensures that the edges with the same label actually get created
+ public void testConcurrentCalls() {
+ final HierarchicalTypeDefinition<ClassType> refType =
+ createClassTypeDef(randomString(), ImmutableSet.<String>of());
+ HierarchicalTypeDefinition<ClassType> type =
+ createClassTypeDef(randomString(), ImmutableSet.<String>of(),
+ new AttributeDefinition("ref", refType.typeName, Multiplicity.OPTIONAL, true, null));
+ try {
+ metadataService.createType(TypesSerialization.toJson(refType, false));
+ metadataService.createType(TypesSerialization.toJson(type, false));
+
+ String refId1 = createBasicEntity(refType);
+ String refId2 = createBasicEntity(refType);
+
+ // Add referenced entity for edge creation
+ final Referenceable instance1 = new Referenceable(type.typeName);
+ instance1.set("ref", new Referenceable(refId1, refType.typeName, null));
+
+ // Add referenced entity for edge creation
+ final Referenceable instance2 = new Referenceable(type.typeName);
+ instance2.set("ref", new Referenceable(refId2, refType.typeName, null));
+
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ List<Future<Object>> futures = new ArrayList<>();
+ // Try parallel creation of both the entities
+ futures.add(executor.submit(new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ return createEntity(instance1);
+ }
+ }));
+ futures.add(executor.submit(new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ return createEntity(instance2);
+ }
+ }));
+ futures.add(executor.submit(new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ return discoveryService.searchByDSL(TestUtils.TABLE_TYPE, new QueryParams(10, 0));
+ }
+ }));
+
+ try {
+ String id1 = (String) futures.get(0).get();
+ String id2 = (String) futures.get(1).get();
+ futures.get(2).get();
+ executor.shutdown();
+
+ assertNotNull(id1);
+ assertNotNull(id2);
+
+ boolean validated1 = assertEdge(id1, type.typeName);
+ boolean validated2 = assertEdge(id2, type.typeName);
+ assertTrue(validated1 && validated2);
+ } catch (InterruptedException | ExecutionException e) {
+ fail("Parallel entity creation failed", e);
+ }
+ } catch (AtlasException e) {
+ fail("Type/Entity creation failed", e);
+ }
+ }
+
+ private String createBasicEntity(final HierarchicalTypeDefinition<ClassType> refType) throws AtlasException {
+ String json = InstanceSerialization.toJson(new Referenceable(refType.typeName), false);
+ CreateUpdateEntitiesResult entities = metadataService.createEntities("[" + json + "]");
+ return entities.getCreatedEntities().get(0);
+ }
+
+ private String createEntity(final Referenceable referenceable) throws AtlasException {
+ String json = InstanceSerialization.toJson(referenceable, false);
+ CreateUpdateEntitiesResult entities = metadataService.createEntities("[" + json + "]");
+ return entities.getCreatedEntities().get(0);
+ }
+
+ private boolean assertEdge(String id, String typeName) throws AtlasException {
+ AtlasGraph graph = TestUtils.getGraph();
+ Iterable<AtlasVertex> vertices = graph.query().has(Constants.GUID_PROPERTY_KEY, id).vertices();
+ AtlasVertex AtlasVertex = vertices.iterator().next();
+ Iterable<AtlasEdge> edges = AtlasVertex.getEdges(AtlasEdgeDirection.OUT, Constants.INTERNAL_PROPERTY_KEY_PREFIX + typeName + ".ref");
+ if (edges.iterator().hasNext()) {
+ ITypedReferenceableInstance entity = metadataService.getEntityDefinition(id);
+ assertNotNull(entity.get("ref"));
+ return true;
+ }
+ return false;
+ }
+
private static class EntitiesChangeListener implements EntityChangeListener {
private List<String> deletedEntities = new ArrayList<>();
private List<String> updatedEntities = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/atlas/blob/d6d349e5/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala b/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
index c844558..f26f98e 100755
--- a/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
+++ b/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
@@ -36,6 +36,8 @@ import org.apache.atlas.repository.MetadataRepository
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.repository.graph.AtlasGraphProvider
import java.net.URL
+
+import org.apache.atlas.`type`.AtlasTypeRegistry
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer
import org.apache.atlas.typesystem.TypesDef
import org.apache.atlas.typesystem.ITypedReferenceableInstance
@@ -74,8 +76,7 @@ trait GraphUtils {
object QueryTestsUtils extends GraphUtils {
def setupTypesAndIndices() : Unit = {
- // FIXME: Do we need to init the AtlasTypeRegistry here ?
- val indexer = new GraphBackedSearchIndexer(null);
+ val indexer = new GraphBackedSearchIndexer(new AtlasTypeRegistry());
val typesDef : TypesDef = defineTypes;
val newTypes = TypeSystem.getInstance.defineTypes(typesDef);
indexer.onAdd(newTypes.values());