You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/04/08 17:01:39 UTC
[atlas] branch branch-2.0 updated: ATLAS-3091: AtlasGraphProvider
Support for Graph with Batch-loading Enable
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 132b9e4 ATLAS-3091: AtlasGraphProvider Support for Graph with Batch-loading Enable
132b9e4 is described below
commit 132b9e4072fe68829acbeb77868f91770fe097f9
Author: Ashutosh Mestry <am...@hortonworks.com>
AuthorDate: Sun Apr 7 23:39:00 2019 -0700
ATLAS-3091: AtlasGraphProvider Support for Graph with Batch-loading Enable
(cherry picked from commit f029a4e024aab36cac60f32c75079357376918ef)
---
.../atlas/repository/graphdb/GraphDatabase.java | 5 ++++
.../repository/graphdb/janus/AtlasJanusGraph.java | 9 +++++--
.../graphdb/janus/AtlasJanusGraphDatabase.java | 5 ++++
.../atlas/repository/graph/AtlasGraphProvider.java | 14 ++++++++++
.../repository/graph/IAtlasGraphProvider.java | 2 ++
.../store/graph/v2/BulkLoadingTest.java} | 30 ++++++++++++++--------
6 files changed, 53 insertions(+), 12 deletions(-)
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
index 3dfc6e8..83f57d1 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
@@ -37,6 +37,11 @@ public interface GraphDatabase<V, E> {
*/
AtlasGraph<V, E> getGraph();
+ /*
+ * Get graph initialized for bulk loading. This instance is used for high-performance ingest.
+ */
+ AtlasGraph<V, E> getGraphBulkLoading();
+
/**
* Sets things up so that getGraph() will return a graph that can be used for running
* tests.
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index 8eea96b..840ebca 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -78,6 +78,7 @@ import java.util.stream.StreamSupport;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
+import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.getGraphInstance;
/**
* Janus implementation of AtlasGraph.
@@ -91,11 +92,15 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
private final StandardJanusGraph janusGraph;
public AtlasJanusGraph() {
+ this(getGraphInstance());
+ }
+
+ public AtlasJanusGraph(JanusGraph graphInstance) {
//determine multi-properties once at startup
JanusGraphManagement mgmt = null;
try {
- mgmt = AtlasJanusGraphDatabase.getGraphInstance().openManagement();
+ mgmt = graphInstance.openManagement();
Iterable<PropertyKey> keys = mgmt.getRelationTypes(PropertyKey.class);
@@ -278,7 +283,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
private JanusGraph getGraph() {
- return AtlasJanusGraphDatabase.getGraphInstance();
+ return getGraphInstance();
}
@Override
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
index 2f367d5..18e6976 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
@@ -264,6 +264,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
return atlasGraphInstance;
}
+ @Override
+ public AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> getGraphBulkLoading() {
+ return new AtlasJanusGraph(getBulkLoadingGraphInstance());
+ }
+
private static void startLocalSolr() {
if (isEmbeddedSolr()) {
try {
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java
index 5aa8033..dfe4b25 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java
@@ -72,6 +72,20 @@ public class AtlasGraphProvider implements IAtlasGraphProvider {
}
}
+ public AtlasGraph getBulkLoading() {
+ try {
+ GraphDatabase<?, ?> graphDB = null;
+ synchronized (AtlasGraphProvider.class) {
+ Class implClass = AtlasRepositoryConfiguration.getGraphDatabaseImpl();
+ graphDB = (GraphDatabase<?, ?>) implClass.newInstance();
+ }
+
+ return graphDB.getGraphBulkLoading();
+ } catch (IllegalAccessException | InstantiationException e) {
+ throw new RuntimeException("Error initializing graph database", e);
+ }
+ }
+
@VisibleForTesting
public static void cleanup() {
getGraphDatabase().cleanup();
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java
index a2cac2d..29add20 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java
@@ -29,4 +29,6 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
public interface IAtlasGraphProvider {
AtlasGraph get() throws RepositoryException;
+
+ AtlasGraph getBulkLoading() throws RepositoryException;
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/BulkLoadingTest.java
old mode 100755
new mode 100644
similarity index 56%
copy from repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java
copy to repository/src/test/java/org/apache/atlas/repository/store/graph/v2/BulkLoadingTest.java
index a2cac2d..fc378ee
--- a/repository/src/main/java/org/apache/atlas/repository/graph/IAtlasGraphProvider.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/BulkLoadingTest.java
@@ -16,17 +16,27 @@
* limitations under the License.
*/
-package org.apache.atlas.repository.graph;
+package org.apache.atlas.repository.store.graph.v2;
+import org.apache.atlas.TestModules;
import org.apache.atlas.repository.RepositoryException;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
-/**
- * Provides a mechanism to control what graph is used in various places. This
- * allows the graph to be mocked out during unit testing and be initialized
- * lazily.
- */
-public interface IAtlasGraphProvider {
-
- AtlasGraph get() throws RepositoryException;
+import javax.inject.Inject;
+
+import static org.testng.Assert.assertNotNull;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class BulkLoadingTest {
+
+ @Inject
+ AtlasGraphProvider graphProvider;
+
+ @Test(expectedExceptions = RuntimeException.class)
+ public void instantiateBulkLoadingForBerkeleyDB() throws RepositoryException {
+ assertNotNull(graphProvider.get());
+ assertNotNull(graphProvider.getBulkLoading());
+ }
}