You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@clerezza.apache.org by mi...@apache.org on 2014/08/07 07:36:42 UTC
git commit: CLEREZZA-930: Improved scalability + clean up
Repository: clerezza
Updated Branches:
refs/heads/master b0cdcfbe4 -> 2d55dbb2f
CLEREZZA-930: Improved scalability + clean up
Project: http://git-wip-us.apache.org/repos/asf/clerezza/repo
Commit: http://git-wip-us.apache.org/repos/asf/clerezza/commit/2d55dbb2
Tree: http://git-wip-us.apache.org/repos/asf/clerezza/tree/2d55dbb2
Diff: http://git-wip-us.apache.org/repos/asf/clerezza/diff/2d55dbb2
Branch: refs/heads/master
Commit: 2d55dbb2f9b0523da0144391e3ae0f6c705e358a
Parents: b0cdcfb
Author: Minto van der Sluis <mi...@apache.org>
Authored: Thu Aug 7 07:36:32 2014 +0200
Committer: Minto van der Sluis <mi...@apache.org>
Committed: Thu Aug 7 07:36:32 2014 +0200
----------------------------------------------------------------------
rdf.virtuoso.storage/pom.xml | 6 +
.../rdf/virtuoso/storage/access/DataAccess.java | 4 -
.../access/VirtuosoWeightedProvider.java | 204 ++-----------------
.../rdf/virtuoso/storage/RdfIOTest.java | 2 -
.../rdf/virtuoso/storage/ThreadSafetyTest.java | 1 -
.../virtuoso/storage/VirtuosoMGraphTest.java | 1 -
.../virtuoso/storage/access/DataAccessTest.java | 1 -
7 files changed, 22 insertions(+), 197 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/clerezza/blob/2d55dbb2/rdf.virtuoso.storage/pom.xml
----------------------------------------------------------------------
diff --git a/rdf.virtuoso.storage/pom.xml b/rdf.virtuoso.storage/pom.xml
index 978f689..14edef5 100644
--- a/rdf.virtuoso.storage/pom.xml
+++ b/rdf.virtuoso.storage/pom.xml
@@ -77,6 +77,12 @@
<artifactId>log4j</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>14.0</version>
+ </dependency>
<!-- TEST (generic) -->
<dependency>
http://git-wip-us.apache.org/repos/asf/clerezza/blob/2d55dbb2/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccess.java
----------------------------------------------------------------------
diff --git a/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccess.java b/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccess.java
index 1af24a4..8b87ea5 100644
--- a/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccess.java
+++ b/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccess.java
@@ -163,10 +163,6 @@ public class DataAccess {
}
}
- public void close() {
-
- }
-
private void close(Object... resources) {
for (Object o : resources) {
if ( o != null ) {
http://git-wip-us.apache.org/repos/asf/clerezza/blob/2d55dbb2/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/VirtuosoWeightedProvider.java
----------------------------------------------------------------------
diff --git a/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/VirtuosoWeightedProvider.java b/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/VirtuosoWeightedProvider.java
index 5fc1bf7..34f97c0 100644
--- a/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/VirtuosoWeightedProvider.java
+++ b/rdf.virtuoso.storage/src/main/java/org/apache/clerezza/rdf/virtuoso/storage/access/VirtuosoWeightedProvider.java
@@ -27,16 +27,14 @@ import java.security.AccessController;
import java.security.PrivilegedAction;
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Collections;
import java.util.Dictionary;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
import javax.sql.PooledConnection;
@@ -47,11 +45,10 @@ import org.apache.clerezza.rdf.core.UriRef;
import org.apache.clerezza.rdf.core.access.EntityAlreadyExistsException;
import org.apache.clerezza.rdf.core.access.EntityUndeletableException;
import org.apache.clerezza.rdf.core.access.NoSuchEntityException;
-import org.apache.clerezza.rdf.core.access.TcManager;
import org.apache.clerezza.rdf.core.access.QueryableTcProvider;
+import org.apache.clerezza.rdf.core.access.TcManager;
import org.apache.clerezza.rdf.core.access.TcProvider;
import org.apache.clerezza.rdf.core.access.WeightedTcProvider;
-import org.apache.clerezza.rdf.virtuoso.storage.VirtuosoGraph;
import org.apache.clerezza.rdf.virtuoso.storage.VirtuosoMGraph;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
@@ -69,6 +66,8 @@ import org.slf4j.LoggerFactory;
import virtuoso.jdbc4.VirtuosoConnectionPoolDataSource;
import virtuoso.jdbc4.VirtuosoException;
+import com.google.common.collect.MapMaker;
+
/**
* A {@link org.apache.clerezza.rdf.core.access.WeightedTcProvider} for
* Virtuoso.
@@ -100,11 +99,8 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
public static final String PASSWORD = "password";
public static final String WEIGHT = "weight";
- // Name of the graph used to contain the registry of the created graphs
- public static final String ACTIVE_GRAPHS_GRAPH = "urn:x-virtuoso:active-graphs";
-
- // Loaded graphs
- private Map<UriRef, VirtuosoMGraph> graphs = new HashMap<UriRef, VirtuosoMGraph>();
+ // Store used graphs in a map that still allows the graph to be garbage collected if not used.
+ private final ConcurrentMap<UriRef, VirtuosoMGraph> graphs = new MapMaker().weakValues().makeMap();
// DataAccess registry
private Set<DataAccess> dataAccessSet = new HashSet<DataAccess>();
@@ -298,17 +294,6 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
throw new ComponentException(e.getLocalizedMessage());
}
}
- // Load remembered graphs
- Set<UriRef> remembered = readRememberedGraphs();
- for (UriRef name : remembered) {
- if (canModify(name)) {
- graphs.put(name, new VirtuosoMGraph(name.getUnicodeString(),
- createDataAccess()));
- } else {
- graphs.put(name, new VirtuosoGraph(name.getUnicodeString(),
- createDataAccess()));
- }
- }
logger.info("Activated VirtuosoWeightedProvider.");
}
@@ -336,156 +321,6 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
return value;
}
- private Set<UriRef> readRememberedGraphs() {
- logger.trace(" readRememberedGraphs()");
- String SQL = "SPARQL SELECT DISTINCT ?G FROM <" + ACTIVE_GRAPHS_GRAPH
- + "> WHERE { ?G a <urn:x-virtuoso/active-graph> }";
- Connection connection = null;
- Exception e = null;
- Statement st = null;
- ResultSet rs = null;
- Set<UriRef> remembered = new HashSet<UriRef>();
- try {
- connection = getConnection();
- st = connection.createStatement();
- logger.debug("Executing SQL: {}", SQL);
- rs = (ResultSet) st.executeQuery(SQL);
- while (rs.next()) {
- UriRef name = new UriRef(rs.getString(1));
- logger.debug(" > Graph {}", name);
- remembered.add(name);
- }
- } catch (VirtuosoException e1) {
- logger.error("Error while executing query/connection.", e1);
- e = e1;
- } catch (SQLException e1) {
- logger.error("Error while executing query/connection.", e1);
- e = e1;
- } finally {
- try {
- if (rs != null)
- rs.close();
- } catch (Exception ex) {
- logger.error("Cannot close result set", ex);
- }
- try {
- if (st != null)
- st.close();
- } catch (Exception ex) {
- logger.error("Cannot close statement", ex);
- }
- try{
- if(connection != null) connection.close();
- }catch (Exception ex) {
- logger.error("Cannot close connection", ex);
- }
- }
- if (e != null) {
- throw new RuntimeException(e);
- }
- return remembered;
- }
-
- private void rememberGraphs(UriRef... graphs) {
- logger.trace(" saveActiveGraphs()");
- if (graphs.length > 0) {
- // Returns the list of graphs in the virtuoso quad store
- String SQL = "SPARQL INSERT INTO <" + ACTIVE_GRAPHS_GRAPH
- + "> { `iri(??)` a <urn:x-virtuoso/active-graph> }";
- Connection connection = null;
- Exception e = null;
- PreparedStatement st = null;
- ResultSet rs = null;
- try {
- try {
- connection = getConnection();
- st = (PreparedStatement) connection
- .prepareStatement(SQL);
- logger.debug("Executing SQL: {}", SQL);
- for (UriRef u : graphs) {
- logger.trace(" > remembering {}", u);
- st.setString(1, u.getUnicodeString());
- st.executeUpdate();
- }
- } catch (Exception e1) {
- logger.error("Error while executing query/connection.", e1);
- e = e1;
- }
- } finally {
- try {
- if (rs != null)
- rs.close();
- } catch (Exception ex) {
- logger.error("Cannot close result set", ex);
- }
- try {
- if (st != null)
- st.close();
- } catch (Exception ex) {
- logger.error("Cannot close statement", ex);
- }
- try{
- if(connection != null) connection.close();
- }catch (Exception ex) {
- logger.error("Cannot close connection", ex);
- }
- }
- if (e != null) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private void forgetGraphs(UriRef... graphs) {
- logger.trace(" forgetGraphs()");
- if (graphs.length > 0) {
- // Returns the list of graphs in the virtuoso quad store
- String SQL = "SPARQL WITH <"
- + ACTIVE_GRAPHS_GRAPH
- + "> DELETE { ?s ?p ?v } WHERE { ?s ?p ?v . FILTER( ?s = iri(??) ) }";
- Exception e = null;
- Connection connection = null;
- PreparedStatement st = null;
- ResultSet rs = null;
- try {
- connection = getConnection();
- st = (PreparedStatement) connection
- .prepareStatement(SQL);
- logger.debug("Executing SQL: {}", SQL);
- for (UriRef u : graphs) {
- logger.trace(" > remembering {}", u);
- st.setString(1, u.getUnicodeString());
- st.executeUpdate();
- }
- } catch (SQLException e1) {
- logger.error("Error while executing query/connection.", e1);
- e = e1;
- } finally {
-
- try {
- if (rs != null)
- rs.close();
- } catch (Exception ex) {
- logger.error("Cannot close result set", ex);
- }
- try {
- if (st != null)
- st.close();
- } catch (Exception ex) {
- logger.error("Cannot close statement", ex);
- }
- try{
- if(connection != null) connection.close();
- }catch (Exception ex) {
- logger.error("Cannot close connection", ex);
- }
- }
- if (e != null) {
- throw new RuntimeException(e);
- }
- }
- }
-
/**
* Deactivates this component.
*
@@ -495,12 +330,6 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
@Deactivate
public void deactivate(ComponentContext cCtx) {
logger.debug("deactivate(ComponentContext {})", cCtx);
- // Save active (possibly empty) graphs to a dedicated graph
- rememberGraphs();
- // XXX Important. Close all opened resources
- for (DataAccess mg : dataAccessSet) {
- mg.close();
- }
try {
pds.close();
} catch (SQLException e) {
@@ -565,7 +394,7 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
logger.debug("getGraph(UriRef {}) ", name);
// If it is read-only, returns the Graph
// If it is not read-only, returns the getGraph() version of the MGraph
- VirtuosoMGraph g = loadGraphOnce(name);
+ VirtuosoMGraph g = retrieveGraph(name);
if (g instanceof Graph) {
return (Graph) g;
} else {
@@ -582,7 +411,7 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
@Override
public MGraph getMGraph(UriRef name) throws NoSuchEntityException {
logger.debug("getMGraph(UriRef {}) ", name);
- VirtuosoMGraph g = loadGraphOnce(name);
+ VirtuosoMGraph g = retrieveGraph(name);
if (g instanceof Graph) {
// We have this graph but only in read-only mode!
throw new NoSuchEntityException(name);
@@ -591,8 +420,8 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
}
/**
- * Load the graph once. It check whether a graph object have been already
- * created for that UriRef, if yes returns it.
+ * Retrieves the graph. First check whether the graph object is already present
+ * in our weak map, if so returns it.
*
* If not check if at least 1 triple is present in the quad for such graph
* identifier. If yes, creates a new graph object and loads it in the map,
@@ -603,10 +432,10 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
* @param name
* @return
*/
- private VirtuosoMGraph loadGraphOnce(UriRef name) {
- logger.debug("loadGraphOnce({})", name);
+ private VirtuosoMGraph retrieveGraph(UriRef name) {
+ logger.debug("retrieveGraph({})", name);
- // Check whether the graph have been already loaded once
+ // Check whether the graph have been already loaded.
if (graphs.containsKey(name)) {
logger.debug("{} is already loaded", name);
return graphs.get(name);
@@ -702,7 +531,7 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
public TripleCollection getTriples(UriRef name)
throws NoSuchEntityException {
logger.debug("getTriples(UriRef {}) ", name);
- return loadGraphOnce(name);
+ return retrieveGraph(name);
}
/**
@@ -870,13 +699,13 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
logger.debug("createVirtuosoMGraph(UriRef {})", name);
// If the graph already exists, we throw an exception
try {
- loadGraphOnce(name);
+ // TODO: check with a query instead of loading?
+ retrieveGraph(name);
throw new EntityAlreadyExistsException(name);
} catch (NoSuchEntityException nsee) {
if (canModify(name)) {
graphs.put(name, new VirtuosoMGraph(name.getUnicodeString(),
createDataAccess()));
- rememberGraphs(name);
return graphs.get(name);
} else {
logger.error("Cannot create MGraph {}", name);
@@ -926,7 +755,6 @@ public class VirtuosoWeightedProvider implements WeightedTcProvider, QueryableTc
} else {
((MGraph) g).clear();
graphs.remove(name);
- forgetGraphs(name);
}
}
http://git-wip-us.apache.org/repos/asf/clerezza/blob/2d55dbb2/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/RdfIOTest.java
----------------------------------------------------------------------
diff --git a/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/RdfIOTest.java b/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/RdfIOTest.java
index 25063b6..f9f3575 100644
--- a/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/RdfIOTest.java
+++ b/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/RdfIOTest.java
@@ -90,8 +90,6 @@ public class RdfIOTest {
log.debug("Clearing graph <{}> of size {}", TEST_GRAPH_NAME,
mgraph.size());
// clear all resources
- da.close();
-
da = null;
mgraph = null;
http://git-wip-us.apache.org/repos/asf/clerezza/blob/2d55dbb2/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/ThreadSafetyTest.java
----------------------------------------------------------------------
diff --git a/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/ThreadSafetyTest.java b/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/ThreadSafetyTest.java
index fd2939f..23a2b61 100644
--- a/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/ThreadSafetyTest.java
+++ b/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/ThreadSafetyTest.java
@@ -84,7 +84,6 @@ public class ThreadSafetyTest {
} finally {
mgraph.clear();
mgraph = null;
- da.close();
da = null;
}
}
http://git-wip-us.apache.org/repos/asf/clerezza/blob/2d55dbb2/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/VirtuosoMGraphTest.java
----------------------------------------------------------------------
diff --git a/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/VirtuosoMGraphTest.java b/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/VirtuosoMGraphTest.java
index 8705fd9..e870f8f 100644
--- a/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/VirtuosoMGraphTest.java
+++ b/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/VirtuosoMGraphTest.java
@@ -517,7 +517,6 @@ public class VirtuosoMGraphTest {
log.info("Clearing graph <{}> of size {}", TEST_GRAPH_NAME,
mgraph.size());
log.debug("Removing graph <{}>", TEST_GRAPH_NAME);
- da.close();
da = null;
mgraph = null;
Statement st = TestUtils.getConnection().createStatement();
http://git-wip-us.apache.org/repos/asf/clerezza/blob/2d55dbb2/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccessTest.java
----------------------------------------------------------------------
diff --git a/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccessTest.java b/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccessTest.java
index 6560628..03e6721 100644
--- a/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccessTest.java
+++ b/rdf.virtuoso.storage/src/test/java/org/apache/clerezza/rdf/virtuoso/storage/access/DataAccessTest.java
@@ -58,7 +58,6 @@ public class DataAccessTest {
@After
public void after() {
da.clearGraph( testGraphName );
- da.close();
da = null;
}