You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/06/18 11:07:31 UTC
git commit: - batched commits now for nodes,
almost doubled the import performance,
PLEASE TRY IF IT WORKS (PostgreSQL and MySQL)
Updated Branches:
refs/heads/develop 5def50274 -> 8e2fc15a8
- batched commits now for nodes, almost doubled the import performance, PLEASE TRY IF IT WORKS (PostgreSQL and MySQL)
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/8e2fc15a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/8e2fc15a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/8e2fc15a
Branch: refs/heads/develop
Commit: 8e2fc15a82b841cfaeeceb1bcd6fef1ee65f4663
Parents: 5def502
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Tue Jun 18 11:07:23 2013 +0200
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Tue Jun 18 11:07:23 2013 +0200
----------------------------------------------------------------------
.../marmotta/kiwi/config/KiWiConfiguration.java | 26 ++
.../kiwi/persistence/KiWiConnection.java | 225 +++++++----
.../kiwi/persistence/KiWiPersistence.java | 25 +-
.../apache/marmotta/kiwi/sail/KiWiStore.java | 2 +
.../marmotta/kiwi/sail/KiWiValueFactory.java | 377 ++++++++++++-------
.../marmotta/kiwi/test/PersistenceTest.java | 38 +-
libraries/kiwi/kiwi-tripletable/pom.xml | 32 ++
.../kiwi/model/caching/TripleTableTest.java | 111 ++++++
.../test/VersioningPersistenceTest.java | 26 +-
.../LDCachingKiWiPersistenceConnection.java | 2 +-
.../services/importer/ImportServiceImpl.java | 18 +-
.../services/triplestore/SesameServiceImpl.java | 2 +
12 files changed, 650 insertions(+), 234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
index 510abb8..f8db80e 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
@@ -65,6 +65,12 @@ public class KiWiConfiguration {
*/
private boolean queryLoggingEnabled = false;
+
+ private boolean batchCommit;
+
+ private int batchSize = 1000;
+
+
public KiWiConfiguration(String name, String jdbcUrl, String dbUser, String dbPassword, KiWiDialect dialect) {
this(name, jdbcUrl, dbUser, dbPassword, dialect, null, null);
}
@@ -77,6 +83,8 @@ public class KiWiConfiguration {
this.name = name;
this.defaultContext = defaultContext;
this.inferredContext = inferredContext;
+
+ batchCommit = dialect.isBatchSupported();
}
@@ -123,4 +131,22 @@ public class KiWiConfiguration {
public void setInferredContext(String inferredContext) {
this.inferredContext = inferredContext;
}
+
+ public boolean isBatchCommit() {
+ return batchCommit;
+ }
+
+ public void setBatchCommit(boolean batchCommit) {
+ if(dialect.isBatchSupported()) {
+ this.batchCommit = batchCommit;
+ }
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
index caad378..7592bfe 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
@@ -796,6 +796,13 @@ public class KiWiConnection {
// }
+
+ public synchronized long getNodeId() throws SQLException {
+ requireJDBCConnection();
+
+ return getNextSequence("seq.nodes");
+ }
+
/**
* Store a new node in the database. The method will retrieve a new database id for the node and update the
* passed object. Afterwards, the node data will be inserted into the database using appropriate INSERT
@@ -803,28 +810,27 @@ public class KiWiConnection {
* <p/>
* If the node already has an ID, the method will do nothing (assuming that it is already persistent)
*
+ *
* @param node
+ * @param batch
* @throws SQLException
*/
- public synchronized void storeNode(KiWiNode node) throws SQLException {
- // if the node already has an ID, storeNode should not be called, since it is already persisted
- if(node.getId() != null) {
- log.warn("node {} already had a node ID, not persisting", node);
- return;
- }
+ public synchronized void storeNode(KiWiNode node, boolean batch) throws SQLException {
// ensure the data type of a literal is persisted first
if(node instanceof KiWiLiteral) {
KiWiLiteral literal = (KiWiLiteral)node;
if(literal.getType() != null && literal.getType().getId() == null) {
- storeNode(literal.getType());
+ storeNode(literal.getType(), batch);
}
}
requireJDBCConnection();
// retrieve a new node id and set it in the node object
- node.setId(getNextSequence("seq.nodes"));
+ if(node.getId() == null) {
+ node.setId(getNextSequence("seq.nodes"));
+ }
// distinguish the different node types and run the appropriate updates
if(node instanceof KiWiUriResource) {
@@ -834,7 +840,12 @@ public class KiWiConnection {
insertNode.setLong(1,node.getId());
insertNode.setString(2,uriResource.stringValue());
insertNode.setTimestamp(3, new Timestamp(uriResource.getCreated().getTime()));
- insertNode.executeUpdate();
+
+ if(batch) {
+ insertNode.addBatch();
+ } else {
+ insertNode.executeUpdate();
+ }
} else if(node instanceof KiWiAnonResource) {
KiWiAnonResource anonResource = (KiWiAnonResource)node;
@@ -843,7 +854,12 @@ public class KiWiConnection {
insertNode.setLong(1,node.getId());
insertNode.setString(2,anonResource.stringValue());
insertNode.setTimestamp(3, new Timestamp(anonResource.getCreated().getTime()));
- insertNode.executeUpdate();
+
+ if(batch) {
+ insertNode.addBatch();
+ } else {
+ insertNode.executeUpdate();
+ }
} else if(node instanceof KiWiDateLiteral) {
KiWiDateLiteral dateLiteral = (KiWiDateLiteral)node;
@@ -857,7 +873,11 @@ public class KiWiConnection {
throw new IllegalStateException("a date literal must have a datatype");
insertNode.setTimestamp(5, new Timestamp(dateLiteral.getCreated().getTime()));
- insertNode.executeUpdate();
+ if(batch) {
+ insertNode.addBatch();
+ } else {
+ insertNode.executeUpdate();
+ }
} else if(node instanceof KiWiIntLiteral) {
KiWiIntLiteral intLiteral = (KiWiIntLiteral)node;
@@ -872,7 +892,11 @@ public class KiWiConnection {
throw new IllegalStateException("an integer literal must have a datatype");
insertNode.setTimestamp(6, new Timestamp(intLiteral.getCreated().getTime()));
- insertNode.executeUpdate();
+ if(batch) {
+ insertNode.addBatch();
+ } else {
+ insertNode.executeUpdate();
+ }
} else if(node instanceof KiWiDoubleLiteral) {
KiWiDoubleLiteral doubleLiteral = (KiWiDoubleLiteral)node;
@@ -886,7 +910,11 @@ public class KiWiConnection {
throw new IllegalStateException("a double literal must have a datatype");
insertNode.setTimestamp(5, new Timestamp(doubleLiteral.getCreated().getTime()));
- insertNode.executeUpdate();
+ if(batch) {
+ insertNode.addBatch();
+ } else {
+ insertNode.executeUpdate();
+ }
} else if(node instanceof KiWiBooleanLiteral) {
KiWiBooleanLiteral booleanLiteral = (KiWiBooleanLiteral)node;
@@ -900,7 +928,11 @@ public class KiWiConnection {
throw new IllegalStateException("a boolean literal must have a datatype");
insertNode.setTimestamp(5, new Timestamp(booleanLiteral.getCreated().getTime()));
- insertNode.executeUpdate();
+ if(batch) {
+ insertNode.addBatch();
+ } else {
+ insertNode.executeUpdate();
+ }
} else if(node instanceof KiWiStringLiteral) {
KiWiStringLiteral stringLiteral = (KiWiStringLiteral)node;
@@ -919,7 +951,11 @@ public class KiWiConnection {
}
insertNode.setTimestamp(5, new Timestamp(stringLiteral.getCreated().getTime()));
- insertNode.executeUpdate();
+ if(batch) {
+ insertNode.addBatch();
+ } else {
+ insertNode.executeUpdate();
+ }
} else {
log.warn("unrecognized node type: {}", node.getClass().getCanonicalName());
}
@@ -928,6 +964,31 @@ public class KiWiConnection {
}
/**
+ * Start a batch operation for inserting nodes. Afterwards, storeNode needs to be called with the batch argument
+ * set to "true".
+ *
+ * @throws SQLException
+ */
+ public void startNodeBatch() throws SQLException {
+ for(String stmt : new String[] { "store.uri", "store.sliteral", "store.bliteral", "store.dliteral", "store.iliteral", "store.tliteral", "store.bnode"}) {
+ PreparedStatement insertNode = getPreparedStatement(stmt);
+ insertNode.clearParameters();
+ insertNode.clearBatch();
+ }
+ }
+
+ /**
+ * Execute the batch operation for inserting nodes into the database.
+ * @throws SQLException
+ */
+ public void commitNodeBatch() throws SQLException {
+ for(String stmt : new String[] { "store.uri", "store.sliteral", "store.bliteral", "store.dliteral", "store.iliteral", "store.tliteral", "store.bnode"}) {
+ PreparedStatement insertNode = getPreparedStatement(stmt);
+ insertNode.executeBatch();
+ }
+ }
+
+ /**
* Store a triple in the database. This method assumes that all nodes used by the triple are already persisted.
*
* @param triple the triple to store
@@ -945,6 +1006,8 @@ public class KiWiConnection {
requireJDBCConnection();
+ boolean hasId = triple.getId() != null;
+
// retrieve a new triple ID and set it in the object
if(triple.getId() == null) {
triple.setId(getNextSequence("seq.triples"));
@@ -952,11 +1015,11 @@ public class KiWiConnection {
if(batchCommit) {
cacheTriple(triple);
- boolean result = tripleBatch.add(triple);
+ tripleBatch.add(triple);
if(tripleBatch.size() >= batchSize) {
flushBatch();
}
- return result;
+ return !hasId;
} else {
try {
PreparedStatement insertTriple = getPreparedStatement("store.triple");
@@ -991,23 +1054,21 @@ public class KiWiConnection {
* @param triple
*/
public void deleteTriple(KiWiTriple triple) throws SQLException {
- if(triple.getId() == null) {
- log.warn("attempting to remove non-persistent triple: {}",triple);
- return;
- }
-
- requireJDBCConnection();
-
- PreparedStatement deleteTriple = getPreparedStatement("delete.triple");
- deleteTriple.setLong(1,triple.getId());
- deleteTriple.executeUpdate();
-
- removeCachedTriple(triple);
-
// make sure the triple is marked as deleted in case some service still holds a reference
triple.setDeleted(true);
triple.setDeletedAt(new Date());
+ if(triple.getId() == null) {
+ log.warn("attempting to remove non-persistent triple: {}", triple);
+ } else {
+ requireJDBCConnection();
+
+ PreparedStatement deleteTriple = getPreparedStatement("delete.triple");
+ deleteTriple.setLong(1,triple.getId());
+ deleteTriple.executeUpdate();
+ }
+ removeCachedTriple(triple);
+
if(tripleBatch != null) {
tripleBatch.remove(triple);
}
@@ -1117,17 +1178,41 @@ public class KiWiConnection {
* @return a new RepositoryResult with a direct connection to the database; the result should be properly closed
* by the caller
*/
- public RepositoryResult<Statement> listTriples(KiWiResource subject, KiWiUriResource predicate, KiWiNode object, KiWiResource context, boolean inferred) throws SQLException {
+ public RepositoryResult<Statement> listTriples(final KiWiResource subject, final KiWiUriResource predicate, final KiWiNode object, final KiWiResource context, final boolean inferred) throws SQLException {
- return new RepositoryResult<Statement>(
- new ExceptionConvertingIteration<Statement, RepositoryException>(listTriplesInternal(subject,predicate,object,context,inferred)) {
- @Override
- protected RepositoryException convert(Exception e) {
- return new RepositoryException("database error while iterating over result set",e);
+
+ if(tripleBatch != null && tripleBatch.size() > 0) {
+ return new RepositoryResult<Statement>(
+ new ExceptionConvertingIteration<Statement, RepositoryException>(
+ new UnionIteration<Statement, SQLException>(
+ new IteratorIteration<Statement, SQLException>(tripleBatch.listTriples(subject,predicate,object,context).iterator()),
+ new DelayedIteration<Statement, SQLException>() {
+ @Override
+ protected Iteration<? extends Statement, ? extends SQLException> createIteration() throws SQLException {
+ return listTriplesInternal(subject,predicate,object,context,inferred);
+ }
+ }
+
+ )
+ ) {
+ @Override
+ protected RepositoryException convert(Exception e) {
+ return new RepositoryException("database error while iterating over result set",e);
+ }
}
- }
- );
+ );
+ } else {
+ return new RepositoryResult<Statement>(
+ new ExceptionConvertingIteration<Statement, RepositoryException>(listTriplesInternal(subject,predicate,object,context,inferred)) {
+ @Override
+ protected RepositoryException convert(Exception e) {
+ return new RepositoryException("database error while iterating over result set",e);
+ }
+ }
+
+ );
+ }
}
/**
@@ -1185,25 +1270,12 @@ public class KiWiConnection {
final ResultSet result = query.executeQuery();
- if(tripleBatch != null && tripleBatch.size() > 0) {
- return new UnionIteration<Statement, SQLException>(
- new IteratorIteration<Statement, SQLException>(tripleBatch.listTriples(subject,predicate,object,context).iterator()),
- new ResultSetIteration<Statement>(result, true, new ResultTransformerFunction<Statement>() {
- @Override
- public Statement apply(ResultSet row) throws SQLException { // could be lazy without even asking the database
- return constructTripleFromDatabase(result);
- }
- })
- );
- } else {
- return new ResultSetIteration<Statement>(result, true, new ResultTransformerFunction<Statement>() {
- @Override
- public Statement apply(ResultSet row) throws SQLException {
- return constructTripleFromDatabase(result);
- }
- });
- }
-
+ return new ResultSetIteration<Statement>(result, true, new ResultTransformerFunction<Statement>() {
+ @Override
+ public Statement apply(ResultSet row) throws SQLException {
+ return constructTripleFromDatabase(result);
+ }
+ });
}
/**
@@ -1677,6 +1749,9 @@ public class KiWiConnection {
*/
public void rollback() throws SQLException {
if(tripleBatch != null && tripleBatch.size() > 0) {
+ for(KiWiTriple triple : tripleBatch) {
+ triple.setId(null);
+ }
tripleBatch.clear();
}
if(connection != null && !connection.isClosed()) {
@@ -1744,32 +1819,42 @@ public class KiWiConnection {
private void flushBatch() throws SQLException {
- if(batchCommit) {
+ if(batchCommit && tripleBatch != null) {
requireJDBCConnection();
commitLock.lock();
try {
+ if(persistence.getValueFactory() != null) {
+ persistence.getValueFactory().flushBatch(this);
+ }
+
PreparedStatement insertTriple = getPreparedStatement("store.triple");
insertTriple.clearParameters();
+ insertTriple.clearBatch();
for(KiWiTriple triple : tripleBatch) {
// retrieve a new triple ID and set it in the object
- if(triple.getId() == null) {
- triple.setId(getNextSequence("seq.triples"));
- log.warn("the batched triple did not have an ID");
- }
+ if(!triple.isDeleted()) {
+ if(triple.getId() == null) {
+ triple.setId(getNextSequence("seq.triples"));
+ log.warn("the batched triple did not have an ID");
+ }
- insertTriple.setLong(1,triple.getId());
- insertTriple.setLong(2,triple.getSubject().getId());
- insertTriple.setLong(3,triple.getPredicate().getId());
- insertTriple.setLong(4,triple.getObject().getId());
- insertTriple.setLong(5,triple.getContext().getId());
- insertTriple.setBoolean(6,triple.isInferred());
- insertTriple.setTimestamp(7, new Timestamp(triple.getCreated().getTime()));
+ insertTriple.setLong(1,triple.getId());
+ insertTriple.setLong(2,triple.getSubject().getId());
+ insertTriple.setLong(3,triple.getPredicate().getId());
+ insertTriple.setLong(4,triple.getObject().getId());
+ insertTriple.setLong(5,triple.getContext().getId());
+ insertTriple.setBoolean(6,triple.isInferred());
+ insertTriple.setTimestamp(7, new Timestamp(triple.getCreated().getTime()));
- insertTriple.addBatch();
+ insertTriple.addBatch();
+ }
}
insertTriple.executeBatch();
tripleBatch.clear();
+ } catch (Throwable ex) {
+ ex.printStackTrace();
+ throw ex;
} finally {
commitLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
index e4206a2..05cd986 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
@@ -23,6 +23,7 @@ import org.apache.marmotta.kiwi.model.rdf.KiWiNode;
import org.apache.marmotta.kiwi.model.rdf.KiWiResource;
import org.apache.marmotta.kiwi.model.rdf.KiWiUriResource;
import org.apache.marmotta.kiwi.persistence.util.ScriptRunner;
+import org.apache.marmotta.kiwi.sail.KiWiValueFactory;
import org.apache.tomcat.jdbc.pool.DataSource;
import org.apache.tomcat.jdbc.pool.PoolProperties;
import org.openrdf.model.Statement;
@@ -67,6 +68,12 @@ public class KiWiPersistence {
*/
private KiWiConfiguration configuration;
+
+ /**
+ * A reference to the value factory used to access this store. Used for notifications when to flush batches.
+ */
+ private KiWiValueFactory valueFactory;
+
@Deprecated
public KiWiPersistence(String name, String jdbcUrl, String db_user, String db_password, KiWiDialect dialect) {
this(new KiWiConfiguration(name,jdbcUrl,db_user,db_password,dialect));
@@ -297,7 +304,12 @@ public class KiWiPersistence {
*/
public KiWiConnection getConnection() throws SQLException {
if(connectionPool != null) {
- return new KiWiConnection(this,configuration.getDialect(),cacheManager);
+ KiWiConnection con = new KiWiConnection(this,configuration.getDialect(),cacheManager);
+ if(getDialect().isBatchSupported()) {
+ con.setBatchCommit(configuration.isBatchCommit());
+ con.setBatchSize(configuration.getBatchSize());
+ }
+ return con;
} else {
throw new SQLException("connection pool is closed, database connections not available");
}
@@ -421,4 +433,15 @@ public class KiWiPersistence {
}
+ public void setValueFactory(KiWiValueFactory valueFactory) {
+ this.valueFactory = valueFactory;
+ }
+
+ public KiWiValueFactory getValueFactory() {
+ return valueFactory;
+ }
+
+ public KiWiConfiguration getConfiguration() {
+ return configuration;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
index b6db0fc..aa945ff 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java
@@ -183,6 +183,7 @@ public class KiWiStore extends NotifyingSailBase {
public void closeValueFactory() {
if(repositoryValueFactory != null) {
repositoryValueFactory = null;
+ persistence.setValueFactory(null);
}
}
@@ -206,6 +207,7 @@ public class KiWiStore extends NotifyingSailBase {
public ValueFactory getValueFactory() {
if(repositoryValueFactory == null) {
repositoryValueFactory = new KiWiValueFactory(this, defaultContext);
+ persistence.setValueFactory(repositoryValueFactory);
}
return repositoryValueFactory;
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java
index 23bd360..db3b5f2 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java
@@ -20,10 +20,7 @@ package org.apache.marmotta.kiwi.sail;
import info.aduna.iteration.Iterations;
import java.sql.SQLException;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
-import java.util.Random;
+import java.util.*;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantLock;
@@ -97,6 +94,19 @@ public class KiWiValueFactory implements ValueFactory {
private String defaultContext;
+ private boolean batchCommit;
+
+ private int batchSize = 1000;
+
+ // the list containing the in-memory nodes that need to be committed later
+ private List<KiWiNode> nodeBatch;
+
+ // a quick lookup allowing to lookup nodes while they are not yet in the database
+ private Map<String,KiWiUriResource> batchUriLookup;
+ private Map<String,KiWiAnonResource> batchBNodeLookup;
+ private Map<String,KiWiLiteral> batchLiteralLookup;
+
+ private ReentrantLock commitLock;
public KiWiValueFactory(KiWiStore store, String defaultContext) {
nodeLock = store.nodeLock;
@@ -109,6 +119,17 @@ public class KiWiValueFactory implements ValueFactory {
this.store = store;
this.defaultContext = defaultContext;
+
+ // batch commits
+ this.nodeBatch = new ArrayList<KiWiNode>(batchSize);
+ this.commitLock = new ReentrantLock();
+
+ this.batchCommit = store.getPersistence().getConfiguration().isBatchCommit();
+ this.batchSize = store.getPersistence().getConfiguration().getBatchSize();
+
+ this.batchUriLookup = new HashMap<String,KiWiUriResource>();
+ this.batchBNodeLookup = new HashMap<String, KiWiAnonResource>();
+ this.batchLiteralLookup = new HashMap<String,KiWiLiteral>();
}
protected KiWiConnection aqcuireConnection() {
@@ -177,25 +198,49 @@ public class KiWiValueFactory implements ValueFactory {
@Override
public URI createURI(String uri) {
- ReentrantLock lock = acquireResourceLock(uri);
- KiWiConnection connection = aqcuireConnection();
- try {
- // first look in the registry for newly created resources if the resource has already been created and
- // is still volatile
- KiWiUriResource result = connection.loadUriResource(uri);
-
- if(result == null) {
- result = new KiWiUriResource(uri);
- connection.storeNode(result);
- }
+ KiWiUriResource result = batchUriLookup.get(uri);
+ if(result != null) {
return result;
- } catch (SQLException e) {
- log.error("database error, could not load URI resource",e);
- throw new IllegalStateException("database error, could not load URI resource",e);
- } finally {
- releaseConnection(connection);
- lock.unlock();
+ } else {
+
+ ReentrantLock lock = acquireResourceLock(uri);
+ KiWiConnection connection = aqcuireConnection();
+ try {
+ // first look in the registry for newly created resources if the resource has already been created and
+ // is still volatile
+ result = connection.loadUriResource(uri);
+
+ if(result == null) {
+ result = new KiWiUriResource(uri);
+
+ if(result.getId() == null) {
+ if(batchCommit) {
+ result.setId(connection.getNodeId());
+ nodeBatch.add(result);
+ batchUriLookup.put(uri,result);
+
+ if(nodeBatch.size() >= batchSize) {
+ flushBatch(connection);
+ }
+ } else {
+ connection.storeNode(result, false);
+ }
+ }
+
+ if(result.getId() == null) {
+ log.error("node ID is null!");
+ }
+ }
+
+ return result;
+ } catch (SQLException e) {
+ log.error("database error, could not load URI resource",e);
+ throw new IllegalStateException("database error, could not load URI resource",e);
+ } finally {
+ releaseConnection(connection);
+ lock.unlock();
+ }
}
}
@@ -226,25 +271,44 @@ public class KiWiValueFactory implements ValueFactory {
*/
@Override
public BNode createBNode(String nodeID) {
- nodeLock.lock();
- KiWiConnection connection = aqcuireConnection();
- try {
- // first look in the registry for newly created resources if the resource has already been created and
- // is still volatile
- KiWiAnonResource result = connection.loadAnonResource(nodeID);
-
- if(result == null) {
- result = new KiWiAnonResource(nodeID);
- connection.storeNode(result);
- }
+ KiWiAnonResource result = batchBNodeLookup.get(nodeID);
+ if(result != null) {
return result;
- } catch (SQLException e) {
- log.error("database error, could not load anonymous resource",e);
- throw new IllegalStateException("database error, could not load anonymous resource",e);
- } finally {
- releaseConnection(connection);
- nodeLock.unlock();
+ } else {
+ nodeLock.lock();
+ KiWiConnection connection = aqcuireConnection();
+ try {
+ // first look in the registry for newly created resources if the resource has already been created and
+ // is still volatile
+ result = connection.loadAnonResource(nodeID);
+
+ if(result == null) {
+ result = new KiWiAnonResource(nodeID);
+
+ if(result.getId() == null) {
+ if(batchCommit) {
+ result.setId(connection.getNodeId());
+ nodeBatch.add(result);
+ batchBNodeLookup.put(nodeID,result);
+
+ if(nodeBatch.size() >= batchSize) {
+ flushBatch(connection);
+ }
+ } else {
+ connection.storeNode(result, false);
+ }
+ }
+ }
+
+ return result;
+ } catch (SQLException e) {
+ log.error("database error, could not load anonymous resource",e);
+ throw new IllegalStateException("database error, could not load anonymous resource",e);
+ } finally {
+ releaseConnection(connection);
+ nodeLock.unlock();
+ }
}
}
@@ -327,112 +391,128 @@ public class KiWiValueFactory implements ValueFactory {
* @return
*/
private <T> KiWiLiteral createLiteral(T value, String lang, String type) {
- if (lang != null) {
- type = LiteralCommons.getRDFLangStringType();
- } else if(type == null) {
- type = LiteralCommons.getXSDType(value.getClass());
- }
-
- KiWiLiteral result = null;
-
- final KiWiUriResource rtype = (KiWiUriResource)createURI(type);
final Locale locale;
if(lang != null) {
locale = LocaleUtils.toLocale(lang.replace("-","_"));
} else
locale = null;
- ReentrantLock lock = acquireLiteralLock(value);
- KiWiConnection connection = aqcuireConnection();
- try {
-
-
- // differentiate between the different types of the value
- if(value instanceof Date || type.equals(Namespaces.NS_XSD+"dateTime")) {
- // parse if necessary
- final Date dvalue;
- if(value instanceof Date) {
- dvalue = (Date)value;
- } else {
- dvalue = DateUtils.parseDate(value.toString());
- }
-
- result = connection.loadLiteral(dvalue);
-
- if(result == null) {
- result= new KiWiDateLiteral(dvalue, rtype);
- }
- } else if(Integer.class.equals(value.getClass()) || int.class.equals(value.getClass()) ||
- Long.class.equals(value.getClass()) || long.class.equals(value.getClass()) ||
- type.equals(Namespaces.NS_XSD+"integer") || type.equals(Namespaces.NS_XSD+"long")) {
- long ivalue = 0;
- if(Integer.class.equals(value.getClass()) || int.class.equals(value.getClass())) {
- ivalue = (Integer)value;
- } else if(Long.class.equals(value.getClass()) || long.class.equals(value.getClass())) {
- ivalue = (Long)value;
- } else {
- ivalue = Long.parseLong(value.toString());
- }
+ if (lang != null) {
+ type = LiteralCommons.getRDFLangStringType();
+ } else if(type == null) {
+ type = LiteralCommons.getXSDType(value.getClass());
+ }
+ KiWiLiteral result = batchLiteralLookup.get(LiteralCommons.createCacheKey(value.toString(),locale,type));
- result = connection.loadLiteral(ivalue);
- if(result == null) {
- result= new KiWiIntLiteral(ivalue, rtype);
- }
- } else if(Double.class.equals(value.getClass()) || double.class.equals(value.getClass()) ||
- Float.class.equals(value.getClass()) || float.class.equals(value.getClass()) ||
- type.equals(Namespaces.NS_XSD+"double") || type.equals(Namespaces.NS_XSD+"float")) {
- double dvalue = 0.0;
- if(Float.class.equals(value.getClass()) || float.class.equals(value.getClass())) {
- dvalue = (Float)value;
- } else if(Double.class.equals(value.getClass()) || double.class.equals(value.getClass())) {
- dvalue = (Double)value;
+ if(result != null) {
+ return result;
+ } else {
+ final KiWiUriResource rtype = (KiWiUriResource)createURI(type);
+
+ ReentrantLock lock = acquireLiteralLock(value);
+ KiWiConnection connection = aqcuireConnection();
+ try {
+
+
+ // differentiate between the different types of the value
+ if(value instanceof Date || type.equals(Namespaces.NS_XSD+"dateTime")) {
+ // parse if necessary
+ final Date dvalue;
+ if(value instanceof Date) {
+ dvalue = (Date)value;
+ } else {
+ dvalue = DateUtils.parseDate(value.toString());
+ }
+
+ result = connection.loadLiteral(dvalue);
+
+ if(result == null) {
+ result= new KiWiDateLiteral(dvalue, rtype);
+ }
+ } else if(Integer.class.equals(value.getClass()) || int.class.equals(value.getClass()) ||
+ Long.class.equals(value.getClass()) || long.class.equals(value.getClass()) ||
+ type.equals(Namespaces.NS_XSD+"integer") || type.equals(Namespaces.NS_XSD+"long")) {
+ long ivalue = 0;
+ if(Integer.class.equals(value.getClass()) || int.class.equals(value.getClass())) {
+ ivalue = (Integer)value;
+ } else if(Long.class.equals(value.getClass()) || long.class.equals(value.getClass())) {
+ ivalue = (Long)value;
+ } else {
+ ivalue = Long.parseLong(value.toString());
+ }
+
+
+ result = connection.loadLiteral(ivalue);
+
+ if(result == null) {
+ result= new KiWiIntLiteral(ivalue, rtype);
+ }
+ } else if(Double.class.equals(value.getClass()) || double.class.equals(value.getClass()) ||
+ Float.class.equals(value.getClass()) || float.class.equals(value.getClass()) ||
+ type.equals(Namespaces.NS_XSD+"double") || type.equals(Namespaces.NS_XSD+"float")) {
+ double dvalue = 0.0;
+ if(Float.class.equals(value.getClass()) || float.class.equals(value.getClass())) {
+ dvalue = (Float)value;
+ } else if(Double.class.equals(value.getClass()) || double.class.equals(value.getClass())) {
+ dvalue = (Double)value;
+ } else {
+ dvalue = Double.parseDouble(value.toString());
+ }
+
+
+ result = connection.loadLiteral(dvalue);
+
+ if(result == null) {
+ result= new KiWiDoubleLiteral(dvalue, rtype);
+ }
+ } else if(Boolean.class.equals(value.getClass()) || boolean.class.equals(value.getClass()) ||
+ type.equals(Namespaces.NS_XSD+"boolean")) {
+ boolean bvalue = false;
+ if(Boolean.class.equals(value.getClass()) || boolean.class.equals(value.getClass())) {
+ bvalue = (Boolean)value;
+ } else {
+ bvalue = Boolean.parseBoolean(value.toString());
+ }
+
+
+ result = connection.loadLiteral(bvalue);
+
+ if(result == null) {
+ result= new KiWiBooleanLiteral(bvalue, rtype);
+ }
} else {
- dvalue = Double.parseDouble(value.toString());
- }
-
-
- result = connection.loadLiteral(dvalue);
+ result = connection.loadLiteral(value.toString(), lang, rtype);
- if(result == null) {
- result= new KiWiDoubleLiteral(dvalue, rtype);
- }
- } else if(Boolean.class.equals(value.getClass()) || boolean.class.equals(value.getClass()) ||
- type.equals(Namespaces.NS_XSD+"boolean")) {
- boolean bvalue = false;
- if(Boolean.class.equals(value.getClass()) || boolean.class.equals(value.getClass())) {
- bvalue = (Boolean)value;
- } else {
- bvalue = Boolean.parseBoolean(value.toString());
+ if(result == null) {
+ result = new KiWiStringLiteral(value.toString(), locale, rtype);
+ }
}
-
- result = connection.loadLiteral(bvalue);
-
- if(result == null) {
- result= new KiWiBooleanLiteral(bvalue, rtype);
+ if(result.getId() == null) {
+ if(batchCommit) {
+ result.setId(connection.getNodeId());
+ nodeBatch.add(result);
+ batchLiteralLookup.put(LiteralCommons.createCacheKey(value.toString(),locale,type), result);
+
+ if(nodeBatch.size() >= batchSize) {
+ flushBatch(connection);
+ }
+ } else {
+ connection.storeNode(result, false);
+ }
}
- } else {
- result = connection.loadLiteral(value.toString(), lang, rtype);
- if(result == null) {
- result = new KiWiStringLiteral(value.toString(), locale, rtype);
- }
- }
+ return result;
- if(result.getId() == null) {
- connection.storeNode(result);
+ } catch (SQLException e) {
+ log.error("database error, could not load literal",e);
+ throw new IllegalStateException("database error, could not load literal",e);
+ } finally {
+ releaseConnection(connection);
+ lock.unlock();
}
-
- return result;
-
- } catch (SQLException e) {
- log.error("database error, could not load literal",e);
- throw new IllegalStateException("database error, could not load literal",e);
- } finally {
- releaseConnection(connection);
- lock.unlock();
}
}
@@ -666,4 +746,47 @@ public class KiWiValueFactory implements ValueFactory {
}
}
+
+ public boolean isBatchCommit() {
+ return batchCommit;
+ }
+
+ public void setBatchCommit(boolean batchCommit) {
+ this.batchCommit = batchCommit;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ /**
+ * Immediately flush the batch to the database. The method expects the underlying connection to start and commit
+ * the node batch.
+ */
+ public void flushBatch(KiWiConnection con) throws SQLException {
+ if(batchCommit && nodeBatch.size() > 0) {
+ commitLock.lock();
+ try {
+ con.startNodeBatch();
+
+ for(KiWiNode n : nodeBatch) {
+ con.storeNode(n,true);
+ }
+
+ con.commitNodeBatch();
+
+ nodeBatch.clear();
+ batchLiteralLookup.clear();
+ batchUriLookup.clear();
+ batchBNodeLookup.clear();
+ } finally {
+ commitLock.unlock();
+ }
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java
index da02000..caceb80 100644
--- a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java
+++ b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java
@@ -197,7 +197,7 @@ public class PersistenceTest {
try {
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiUriResource uri = new KiWiUriResource("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
- connection.storeNode(uri);
+ connection.storeNode(uri, false);
// check if it then has a database ID
Assert.assertNotNull(uri.getId());
@@ -272,7 +272,7 @@ public class PersistenceTest {
try {
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiAnonResource bnode = new KiWiAnonResource(RandomStringUtils.randomAlphanumeric(8));
- connection.storeNode(bnode);
+ connection.storeNode(bnode, false);
// check if it then has a database ID
Assert.assertNotNull(bnode.getId());
@@ -346,11 +346,11 @@ public class PersistenceTest {
KiWiConnection connection = persistence.getConnection();
try {
KiWiUriResource stype = new KiWiUriResource(Namespaces.NS_XSD+"string");
- connection.storeNode(stype);
+ connection.storeNode(stype, false);
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiStringLiteral literal = new KiWiStringLiteral(RandomStringUtils.randomAlphanumeric(8),null,stype);
- connection.storeNode(literal);
+ connection.storeNode(literal, false);
// check if it then has a database ID
Assert.assertNotNull(literal.getId());
@@ -425,11 +425,11 @@ public class PersistenceTest {
KiWiConnection connection = persistence.getConnection();
try {
KiWiUriResource stype = new KiWiUriResource(getRDFLangStringType());
- connection.storeNode(stype);
+ connection.storeNode(stype, false);
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiStringLiteral literal = new KiWiStringLiteral(RandomStringUtils.randomAlphanumeric(8), Locale.ENGLISH, stype);
- connection.storeNode(literal);
+ connection.storeNode(literal, false);
// check if it then has a database ID
Assert.assertNotNull(literal.getId());
@@ -507,7 +507,7 @@ public class PersistenceTest {
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiStringLiteral literal = new KiWiStringLiteral(RandomStringUtils.randomAlphanumeric(8), null, uri);
- connection.storeNode(literal);
+ connection.storeNode(literal, false);
// check if it then has a database ID
Assert.assertNotNull(literal.getId());
@@ -593,7 +593,7 @@ public class PersistenceTest {
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiIntLiteral literal = new KiWiIntLiteral(value, uri);
- connection.storeNode(literal);
+ connection.storeNode(literal, false);
// check if it then has a database ID
Assert.assertNotNull(literal.getId());
@@ -698,7 +698,7 @@ public class PersistenceTest {
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiDoubleLiteral literal = new KiWiDoubleLiteral(value, uri);
- connection.storeNode(literal);
+ connection.storeNode(literal, false);
// check if it then has a database ID
Assert.assertNotNull(literal.getId());
@@ -802,7 +802,7 @@ public class PersistenceTest {
boolean value = rnd.nextBoolean();
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiBooleanLiteral literal = new KiWiBooleanLiteral(value, uri);
- connection.storeNode(literal);
+ connection.storeNode(literal, false);
// check if it then has a database ID
Assert.assertNotNull(literal.getId());
@@ -905,7 +905,7 @@ public class PersistenceTest {
Date value = new Date();
// add a new URI to the triple store and check if it exists afterwards, before and after commit
KiWiDateLiteral literal = new KiWiDateLiteral(value, uri);
- connection.storeNode(literal);
+ connection.storeNode(literal, false);
// check if it then has a database ID
Assert.assertNotNull(literal.getId());
@@ -1009,13 +1009,13 @@ public class PersistenceTest {
KiWiStringLiteral object_2 = new KiWiStringLiteral(RandomStringUtils.randomAlphanumeric(32),null,stype);
KiWiUriResource context = new KiWiUriResource("http://localhost/context/"+RandomStringUtils.randomAlphanumeric(8));
- connection.storeNode(stype);
- connection.storeNode(subject);
- connection.storeNode(pred_1);
- connection.storeNode(pred_2);
- connection.storeNode(object_1);
- connection.storeNode(object_2);
- connection.storeNode(context);
+ connection.storeNode(stype, false);
+ connection.storeNode(subject, false);
+ connection.storeNode(pred_1, false);
+ connection.storeNode(pred_2, false);
+ connection.storeNode(object_1, false);
+ connection.storeNode(object_2, false);
+ connection.storeNode(context, false);
KiWiTriple triple1 = new KiWiTriple(subject,pred_1,object_1,context);
KiWiTriple triple2 = new KiWiTriple(subject,pred_2,object_2,context);
@@ -1054,7 +1054,7 @@ public class PersistenceTest {
Assert.assertEquals(0, connection.getSize(subject));
// test database contents
- PreparedStatement stmt = connection.getJDBCConnection().prepareStatement("SELECT * FROM triples WHERE deleted = false");
+ PreparedStatement stmt = connection.getJDBCConnection().prepareStatement("SELECT * FROM triples WHERE deleted = false ORDER BY subject, predicate");
ResultSet dbResult1 = stmt.executeQuery();
Assert.assertTrue(dbResult1.next());
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-tripletable/pom.xml
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-tripletable/pom.xml b/libraries/kiwi/kiwi-tripletable/pom.xml
index 3aa74e9..845c4c2 100644
--- a/libraries/kiwi/kiwi-tripletable/pom.xml
+++ b/libraries/kiwi/kiwi-tripletable/pom.xml
@@ -47,5 +47,37 @@
<artifactId>guava</artifactId>
</dependency>
+ <dependency>
+ <artifactId>junit</artifactId>
+ <groupId>junit</groupId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <artifactId>hamcrest-core</artifactId>
+ <groupId>org.hamcrest</groupId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <artifactId>hamcrest-library</artifactId>
+ <groupId>org.hamcrest</groupId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-tripletable/src/test/java/org/apache/marmotta/kiwi/model/caching/TripleTableTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-tripletable/src/test/java/org/apache/marmotta/kiwi/model/caching/TripleTableTest.java b/libraries/kiwi/kiwi-tripletable/src/test/java/org/apache/marmotta/kiwi/model/caching/TripleTableTest.java
new file mode 100644
index 0000000..1fa2e32
--- /dev/null
+++ b/libraries/kiwi/kiwi-tripletable/src/test/java/org/apache/marmotta/kiwi/model/caching/TripleTableTest.java
@@ -0,0 +1,111 @@
+package org.apache.marmotta.kiwi.model.caching;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+import org.openrdf.model.Literal;
+import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
+import org.openrdf.model.impl.LiteralImpl;
+import org.openrdf.model.impl.StatementImpl;
+import org.openrdf.model.impl.URIImpl;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test cases for triple tables.
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class TripleTableTest {
+
+
+
+ @Test
+ public void testListTriples() {
+ URI subject1 = new URIImpl("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+ URI subject2 = new URIImpl("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+ URI predicate1 = new URIImpl("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+ URI predicate2 = new URIImpl("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+ Literal object1 = new LiteralImpl("http://localhost/"+ RandomStringUtils.random(40));
+ Literal object2 = new LiteralImpl("http://localhost/"+ RandomStringUtils.random(40));
+
+ Statement stmt1 = new StatementImpl(subject1,predicate1,object1);
+ Statement stmt2 = new StatementImpl(subject1,predicate1,object2);
+ Statement stmt3 = new StatementImpl(subject1,predicate2,object1);
+ Statement stmt4 = new StatementImpl(subject1,predicate2,object2);
+ Statement stmt5 = new StatementImpl(subject2,predicate1,object1);
+ Statement stmt6 = new StatementImpl(subject2,predicate1,object2);
+ Statement stmt7 = new StatementImpl(subject2,predicate2,object1);
+ Statement stmt8 = new StatementImpl(subject2,predicate2,object2);
+
+ TripleTable<Statement> table = new TripleTable<>();
+ table.add(stmt1);
+ table.add(stmt2);
+ table.add(stmt3);
+ table.add(stmt4);
+ table.add(stmt5);
+ table.add(stmt6);
+ table.add(stmt7);
+ //table.add(stmt8);
+
+ // tests
+
+ // 1. test existence and non-existence of a triple
+ assertEquals(1, table.listTriples(subject2,predicate2,object1,null).size());
+ assertEquals(0, table.listTriples(subject2,predicate2,object2,null).size());
+
+ // 2. test listing with wildcards
+ assertEquals(7, table.listTriples(null,null,null,null).size());
+ assertEquals(4, table.listTriples(subject1,null,null,null).size());
+ assertEquals(3, table.listTriples(subject2,null,null,null).size());
+ assertEquals(4, table.listTriples(null,predicate1,null,null).size());
+ assertEquals(3, table.listTriples(null,predicate2,null,null).size());
+ assertEquals(4, table.listTriples(null,null,object1,null).size());
+ assertEquals(3, table.listTriples(null,null,object2,null).size());
+ assertEquals(2, table.listTriples(subject1,predicate1,null,null).size());
+ assertEquals(1, table.listTriples(subject2,predicate2,null,null).size());
+ }
+
+ @Test
+ public void testRemoveTriples() {
+ URI subject1 = new URIImpl("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+ URI subject2 = new URIImpl("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+ URI predicate1 = new URIImpl("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+ URI predicate2 = new URIImpl("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+ Literal object1 = new LiteralImpl("http://localhost/"+ RandomStringUtils.random(40));
+ Literal object2 = new LiteralImpl("http://localhost/"+ RandomStringUtils.random(40));
+
+ Statement stmt1 = new StatementImpl(subject1,predicate1,object1);
+ Statement stmt2 = new StatementImpl(subject1,predicate1,object2);
+ Statement stmt3 = new StatementImpl(subject1,predicate2,object1);
+ Statement stmt4 = new StatementImpl(subject1,predicate2,object2);
+ Statement stmt5 = new StatementImpl(subject2,predicate1,object1);
+ Statement stmt6 = new StatementImpl(subject2,predicate1,object2);
+ Statement stmt7 = new StatementImpl(subject2,predicate2,object1);
+ Statement stmt8 = new StatementImpl(subject2,predicate2,object2);
+
+ TripleTable<Statement> table = new TripleTable<>();
+ table.add(stmt1);
+ table.add(stmt2);
+ table.add(stmt3);
+ table.add(stmt4);
+ table.add(stmt5);
+ table.add(stmt6);
+ table.add(stmt7);
+ table.add(stmt8);
+
+ // tests
+
+ // 1. test existence and non-existence of a triple
+ assertEquals(1, table.listTriples(subject2,predicate2,object1,null).size());
+ assertEquals(1, table.listTriples(subject2,predicate2,object2,null).size());
+
+
+ table.remove(stmt8);
+
+ assertEquals(1, table.listTriples(subject2,predicate2,object1,null).size());
+ assertEquals(0, table.listTriples(subject2,predicate2,object2,null).size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/kiwi/kiwi-versioning/src/test/java/org/apache/marmotta/kiwi/versioning/test/VersioningPersistenceTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-versioning/src/test/java/org/apache/marmotta/kiwi/versioning/test/VersioningPersistenceTest.java b/libraries/kiwi/kiwi-versioning/src/test/java/org/apache/marmotta/kiwi/versioning/test/VersioningPersistenceTest.java
index b6f8e4a..a7d6ad4 100644
--- a/libraries/kiwi/kiwi-versioning/src/test/java/org/apache/marmotta/kiwi/versioning/test/VersioningPersistenceTest.java
+++ b/libraries/kiwi/kiwi-versioning/src/test/java/org/apache/marmotta/kiwi/versioning/test/VersioningPersistenceTest.java
@@ -195,13 +195,13 @@ public class VersioningPersistenceTest {
KiWiStringLiteral object_2 = new KiWiStringLiteral(RandomStringUtils.randomAlphanumeric(32));
KiWiUriResource context = new KiWiUriResource("http://localhost/context/"+RandomStringUtils.randomAlphanumeric(8));
- connection.storeNode(subject1);
- connection.storeNode(subject2);
- connection.storeNode(pred_1);
- connection.storeNode(pred_2);
- connection.storeNode(object_1);
- connection.storeNode(object_2);
- connection.storeNode(context);
+ connection.storeNode(subject1, false);
+ connection.storeNode(subject2, false);
+ connection.storeNode(pred_1, false);
+ connection.storeNode(pred_2, false);
+ connection.storeNode(object_1, false);
+ connection.storeNode(object_2, false);
+ connection.storeNode(context, false);
KiWiTriple triple1 = new KiWiTriple(subject1,pred_1,object_1,context);
KiWiTriple triple2 = new KiWiTriple(subject2,pred_2,object_2,context);
@@ -271,12 +271,12 @@ public class VersioningPersistenceTest {
KiWiStringLiteral object_2 = new KiWiStringLiteral(RandomStringUtils.randomAlphanumeric(32));
KiWiUriResource context = new KiWiUriResource("http://localhost/context/"+RandomStringUtils.randomAlphanumeric(8));
- connection.storeNode(subject);
- connection.storeNode(pred_1);
- connection.storeNode(pred_2);
- connection.storeNode(object_1);
- connection.storeNode(object_2);
- connection.storeNode(context);
+ connection.storeNode(subject, false);
+ connection.storeNode(pred_1, false);
+ connection.storeNode(pred_2, false);
+ connection.storeNode(object_1, false);
+ connection.storeNode(object_2, false);
+ connection.storeNode(context, false);
KiWiTriple triple1 = new KiWiTriple(subject,pred_1,object_1,context);
KiWiTriple triple2 = new KiWiTriple(subject,pred_2,object_2,context);
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/libraries/ldcache/ldcache-backend-kiwi/src/main/java/org/apache/marmotta/ldcache/backend/kiwi/persistence/LDCachingKiWiPersistenceConnection.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-backend-kiwi/src/main/java/org/apache/marmotta/ldcache/backend/kiwi/persistence/LDCachingKiWiPersistenceConnection.java b/libraries/ldcache/ldcache-backend-kiwi/src/main/java/org/apache/marmotta/ldcache/backend/kiwi/persistence/LDCachingKiWiPersistenceConnection.java
index d0e8675..9ea30ed 100644
--- a/libraries/ldcache/ldcache-backend-kiwi/src/main/java/org/apache/marmotta/ldcache/backend/kiwi/persistence/LDCachingKiWiPersistenceConnection.java
+++ b/libraries/ldcache/ldcache-backend-kiwi/src/main/java/org/apache/marmotta/ldcache/backend/kiwi/persistence/LDCachingKiWiPersistenceConnection.java
@@ -329,7 +329,7 @@ public class LDCachingKiWiPersistenceConnection {
* @throws java.sql.SQLException
*/
public void storeNode(KiWiNode node) throws SQLException {
- connection.storeNode(node);
+ connection.storeNode(node, false);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/importer/ImportServiceImpl.java b/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/importer/ImportServiceImpl.java
index 5d088ce..0a65d84 100644
--- a/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/importer/ImportServiceImpl.java
+++ b/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/importer/ImportServiceImpl.java
@@ -71,17 +71,29 @@ public class ImportServiceImpl implements ImportService{
@Override
public int importData(URL url, String format, Resource user, URI context) throws MarmottaImportException {
- return getImporterInstance(format).importData(url,format,user,context);
+ long start = System.currentTimeMillis();
+ int result = getImporterInstance(format).importData(url,format,user,context);
+ long end = System.currentTimeMillis();
+ log.info("data import finished ({} ms}", end-start);
+ return result;
}
@Override
public int importData(InputStream is, String format, Resource user, URI context) throws MarmottaImportException {
- return getImporterInstance(format).importData(is,format,user,context);
+ long start = System.currentTimeMillis();
+ int result = getImporterInstance(format).importData(is,format,user,context);
+ long end = System.currentTimeMillis();
+ log.info("data import finished ({} ms}", end-start);
+ return result;
}
@Override
public int importData(Reader reader, String format, Resource user, URI context) throws MarmottaImportException {
- return getImporterInstance(format).importData(reader,format,user,context);
+ long start = System.currentTimeMillis();
+ int result = getImporterInstance(format).importData(reader,format,user,context);
+ long end = System.currentTimeMillis();
+ log.info("data import finished ({} ms}", end-start);
+ return result;
}
private Importer getImporterInstance(String type) throws MarmottaImportException {
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/8e2fc15a/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java
----------------------------------------------------------------------
diff --git a/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java b/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java
index 9c60ab4..72dc965 100644
--- a/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java
+++ b/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java
@@ -148,9 +148,11 @@ public class SesameServiceImpl implements SesameService {
String jdbcUrl = configurationService.getStringConfiguration("database.url");
String dbUser = configurationService.getStringConfiguration("database.user");
String dbPass = configurationService.getStringConfiguration("database.password");
+ boolean batchCommit = configurationService.getBooleanConfiguration("database.batchcommit", true);
KiWiConfiguration configuration = new KiWiConfiguration("lmf", jdbcUrl, dbUser, dbPass, dialect, configurationService.getDefaultContext(), configurationService.getInferredContext());
configuration.setQueryLoggingEnabled(configurationService.getBooleanConfiguration("database.debug.slowqueries",false));
+ configuration.setBatchCommit(batchCommit);
store = new KiWiStore(configuration);