You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/09/13 12:23:01 UTC
git commit: fixing some test errors, perparing for release
Updated Branches:
refs/heads/develop ec26977c9 -> c7563c0d7
fixing some test errors, perparing for release
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/c7563c0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/c7563c0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/c7563c0d
Branch: refs/heads/develop
Commit: c7563c0d7d1030dc60b5e7682ac7cae0adf34331
Parents: ec26977
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Fri Sep 13 12:22:55 2013 +0200
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Fri Sep 13 12:22:55 2013 +0200
----------------------------------------------------------------------
.../concurrent/ConcurrentFacadingTest.java | 3 +
.../kiwi/reasoner/engine/ReasoningEngine.java | 42 ++++++--
.../test/engine/ReasoningEngineTest.java | 2 +-
.../reasoner/test/sail/ReasoningSailTest.java | 1 +
.../KiWiRDFSchemaRepositoryConnectionTest.java | 2 +-
.../kiwi/persistence/KiWiConnection.java | 108 +++++++++++--------
.../kiwi/persistence/h2/statements.properties | 4 +-
.../kiwi/test/sesame/KiWiLocaleTest.java | 5 +-
8 files changed, 109 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
----------------------------------------------------------------------
diff --git a/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java b/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
index 504699d..03cdc09 100644
--- a/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
+++ b/commons/sesame-tools-facading/src/test/java/org/apache/marmotta/commons/sesame/facading/concurrent/ConcurrentFacadingTest.java
@@ -1,5 +1,6 @@
package org.apache.marmotta.commons.sesame.facading.concurrent;
+import java.util.ConcurrentModificationException;
import java.util.UUID;
import org.apache.marmotta.commons.sesame.facading.AbstractFacadingTest;
@@ -169,6 +170,8 @@ public class ConcurrentFacadingTest extends AbstractFacadingTest {
subCon_2.close();
}
}
+ } catch (ConcurrentModificationException ex) {
+ // do nothing, H2 locking
} finally {
if (mainCon.isActive()) mainCon.rollback();
mainCon.close();
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java b/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
index 18e0607..47891c6 100644
--- a/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
+++ b/libraries/kiwi/kiwi-reasoner/src/main/java/org/apache/marmotta/kiwi/reasoner/engine/ReasoningEngine.java
@@ -121,6 +121,7 @@ public class ReasoningEngine implements TransactionListener {
*/
private static long taskCounter = 0;
+ private boolean isshutdown = false;
/**
* The worker thread for the reasoner.
@@ -894,17 +895,44 @@ public class ReasoningEngine implements TransactionListener {
}
public void shutdown() {
- log.info("shutting down reasoning service ...");
+ shutdown(false);
+ }
- for(int i = 0; i<20 && isRunning(); i++) {
- log.warn("reasoner not yet finished, waiting for 1 seconds (try={})", i+1);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
+ public void shutdown(boolean force) {
+ if(isshutdown)
+ return;
+
+ if(force) {
+ log.warn("forced shutdown of reasoning service initiated, state will be inconsistent ...");
+
+ reasonerThread.shutdown();
+
+ for(int i = 0; i<3 && isRunning(); i++) {
+ log.warn("reasoner not yet finished, waiting for 1 seconds (try={})", i+1);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
}
+
+ // yes, I know it is unsafe; it is only used when forcefully shutting down on test ends before the database is deleted...
+ reasonerThread.stop();
+
+ } else {
+ log.info("graceful shutdown of reasoning service initiated ...");
+
+ for(int i = 0; i<20 && isRunning(); i++) {
+ log.warn("reasoner not yet finished, waiting for 1 seconds (try={})", i+1);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ }
+
+ reasonerThread.shutdown();
}
- reasonerThread.shutdown();
+ isshutdown = true;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
index 75b7cad..fb2c497 100644
--- a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
+++ b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/engine/ReasoningEngineTest.java
@@ -129,7 +129,7 @@ public class ReasoningEngineTest {
@After
public void dropDatabase() throws Exception {
- engine.shutdown();
+ engine.shutdown(true);
rpersistence.dropDatabase();
persistence.dropDatabase();
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
index 15f4d8f..64da3c1 100644
--- a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
+++ b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sail/ReasoningSailTest.java
@@ -88,6 +88,7 @@ public class ReasoningSailTest {
@After
public void dropDatabase() throws Exception {
+ rsail.getEngine().shutdown(true);
rsail.getPersistence().dropDatabase();
store.getPersistence().dropDatabase();
repository.shutDown();
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
index 8d959ec..90bf0f9 100644
--- a/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
+++ b/libraries/kiwi/kiwi-reasoner/src/test/java/org/apache/marmotta/kiwi/reasoner/test/sesame/KiWiRDFSchemaRepositoryConnectionTest.java
@@ -80,7 +80,7 @@ public class KiWiRDFSchemaRepositoryConnectionTest extends RDFSchemaRepositoryCo
Sail wsail = new SailWrapper(rsail) {
@Override
public void shutDown() throws SailException {
- rsail.getEngine().shutdown();
+ rsail.getEngine().shutdown(true);
try {
rsail.getPersistence().dropDatabase();
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
index 119256c..9c7de67 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
@@ -28,7 +28,6 @@ import com.google.common.base.Preconditions;
import net.sf.ehcache.Cache;
import net.sf.ehcache.Element;
-import org.apache.commons.lang3.LocaleUtils;
import org.apache.marmotta.kiwi.caching.KiWiCacheManager;
import org.apache.marmotta.kiwi.config.KiWiConfiguration;
import org.apache.marmotta.kiwi.model.caching.TripleTable;
@@ -46,7 +45,6 @@ import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.*;
import java.util.Date;
-import java.util.concurrent.Executor;
import java.util.concurrent.locks.ReentrantLock;
/**
@@ -1062,7 +1060,7 @@ public class KiWiConnection {
* @throws NullPointerException in case the subject, predicate, object or context have not been persisted
* @return true in case the update added a new triple to the database, false in case the triple already existed
*/
- public synchronized boolean storeTriple(KiWiTriple triple) throws SQLException {
+ public synchronized boolean storeTriple(final KiWiTriple triple) throws SQLException {
// mutual exclusion: prevent parallel adding and removing of the same triple
synchronized (triple) {
@@ -1104,30 +1102,39 @@ public class KiWiConnection {
try {
- PreparedStatement insertTriple = getPreparedStatement("store.triple");
- insertTriple.setLong(1,triple.getId());
- insertTriple.setLong(2,triple.getSubject().getId());
- insertTriple.setLong(3,triple.getPredicate().getId());
- insertTriple.setLong(4,triple.getObject().getId());
- if(triple.getContext() != null) {
- insertTriple.setLong(5,triple.getContext().getId());
- } else {
- insertTriple.setNull(5, Types.BIGINT);
- }
- insertTriple.setBoolean(6,triple.isInferred());
- insertTriple.setTimestamp(7, new Timestamp(triple.getCreated().getTime()));
- int count = insertTriple.executeUpdate();
+ RetryExecution<Boolean> execution = new RetryExecution<>("STORE");
+ execution.setUseSavepoint(true);
+ execution.execute(connection, new RetryCommand<Boolean>() {
+ @Override
+ public Boolean run() throws SQLException {
+ PreparedStatement insertTriple = getPreparedStatement("store.triple");
+ insertTriple.setLong(1,triple.getId());
+ insertTriple.setLong(2,triple.getSubject().getId());
+ insertTriple.setLong(3,triple.getPredicate().getId());
+ insertTriple.setLong(4,triple.getObject().getId());
+ if(triple.getContext() != null) {
+ insertTriple.setLong(5,triple.getContext().getId());
+ } else {
+ insertTriple.setNull(5, Types.BIGINT);
+ }
+ insertTriple.setBoolean(6,triple.isInferred());
+ insertTriple.setTimestamp(7, new Timestamp(triple.getCreated().getTime()));
+ int count = insertTriple.executeUpdate();
- cacheTriple(triple);
+ cacheTriple(triple);
+
+ return count > 0;
+ }
+ });
+
+ return !hasId;
- return count > 0;
} catch(SQLException ex) {
if("HYT00".equals(ex.getSQLState())) { // H2 table locking timeout
throw new ConcurrentModificationException("the same triple was modified in concurrent transactions (triple="+triple+")");
+ } else {
+ throw ex;
}
- // this is an ugly hack to catch duplicate key errors in some databases (H2)
- // better option could be http://stackoverflow.com/questions/6736518/h2-java-insert-ignore-allow-exception
- return false;
}
}
}
@@ -1192,9 +1199,9 @@ public class KiWiConnection {
RetryExecution execution = new RetryExecution("DELETE");
execution.setUseSavepoint(true);
- execution.execute(connection, new RetryCommand() {
+ execution.execute(connection, new RetryCommand<Void>() {
@Override
- public void run() throws SQLException {
+ public Void run() throws SQLException {
// mutual exclusion: prevent parallel adding and removing of the same triple
synchronized (triple) {
@@ -1202,19 +1209,19 @@ public class KiWiConnection {
triple.setDeleted(true);
triple.setDeletedAt(new Date());
- if(triple.getId() == null) {
+ if (triple.getId() == null) {
log.warn("attempting to remove non-persistent triple: {}", triple);
removeCachedTriple(triple);
} else {
- if(batchCommit) {
+ if (batchCommit) {
// need to remove from triple batch and from database
commitLock.lock();
try {
- if(tripleBatch == null || !tripleBatch.remove(triple)) {
+ if (tripleBatch == null || !tripleBatch.remove(triple)) {
PreparedStatement deleteTriple = getPreparedStatement("delete.triple");
synchronized (deleteTriple) {
- deleteTriple.setLong(1,triple.getId());
+ deleteTriple.setLong(1, triple.getId());
deleteTriple.executeUpdate();
}
deletedStatementsLog.add(triple.getId());
@@ -1227,7 +1234,7 @@ public class KiWiConnection {
PreparedStatement deleteTriple = getPreparedStatement("delete.triple");
synchronized (deleteTriple) {
- deleteTriple.setLong(1,triple.getId());
+ deleteTriple.setLong(1, triple.getId());
deleteTriple.executeUpdate();
}
deletedStatementsLog.add(triple.getId());
@@ -1237,6 +1244,8 @@ public class KiWiConnection {
removeCachedTriple(triple);
}
}
+
+ return null;
}
});
@@ -1789,7 +1798,7 @@ public class KiWiConnection {
private void cacheNode(KiWiNode node) {
if(node.getId() != null) {
- nodeCache.put(new Element(node.getId(),node));
+ nodeCache.put(new Element(node.getId(), node));
}
if(node instanceof KiWiUriResource) {
uriCache.put(new Element(node.stringValue(), node));
@@ -1976,9 +1985,9 @@ public class KiWiConnection {
numberOfCommits++;
RetryExecution execution = new RetryExecution("COMMIT");
- execution.execute(connection, new RetryCommand() {
+ execution.execute(connection, new RetryCommand<Void>() {
@Override
- public void run() throws SQLException {
+ public Void run() throws SQLException {
if(persistence.getConfiguration().isCommitSequencesOnCommit() || numberOfCommits % 100 == 0) {
commitMemorySequences();
}
@@ -1994,6 +2003,8 @@ public class KiWiConnection {
if(connection != null) {
connection.commit();
}
+
+ return null;
}
});
}
@@ -2135,9 +2146,9 @@ public class KiWiConnection {
RetryExecution execution = new RetryExecution("FLUSH BATCH");
execution.setUseSavepoint(true);
- execution.execute(connection, new RetryCommand() {
+ execution.execute(connection, new RetryCommand<Void>() {
@Override
- public void run() throws SQLException {
+ public Void run() throws SQLException {
PreparedStatement insertTriple = getPreparedStatement("store.triple");
insertTriple.clearParameters();
insertTriple.clearBatch();
@@ -2177,6 +2188,7 @@ public class KiWiConnection {
tripleBatch.clear();
+ return null;
}
});
@@ -2189,9 +2201,9 @@ public class KiWiConnection {
}
- protected static interface RetryCommand {
+ protected static interface RetryCommand<T> {
- public void run() throws SQLException;
+ public T run() throws SQLException;
}
/**
@@ -2199,7 +2211,7 @@ public class KiWiConnection {
* and should be retried several times before giving up completely.
*
*/
- protected static class RetryExecution {
+ protected static class RetryExecution<T> {
// counter for current number of retries
private int retries = 0;
@@ -2260,36 +2272,40 @@ public class KiWiConnection {
return sqlStates;
}
- public void execute(Connection connection, RetryCommand command) throws SQLException {
+ public T execute(Connection connection, RetryCommand<T> command) throws SQLException {
Savepoint savepoint = null;
if(useSavepoint) {
savepoint = connection.setSavepoint();
}
try {
- command.run();
+ T result = command.run();
if(useSavepoint && savepoint != null) {
connection.releaseSavepoint(savepoint);
}
+
+ return result;
} catch (SQLException ex) {
if(retries < maxRetries && (sqlStates.size() == 0 || sqlStates.contains(ex.getSQLState()))) {
if(useSavepoint && savepoint != null) {
connection.rollback(savepoint);
}
- log.warn("{}: temporary conflict, retrying in {} ms ... (thread={}, retry={})", name, retryInterval, Thread.currentThread().getName(), retries);
+ Random rnd = new Random();
+ long sleep = retryInterval - 250 + rnd.nextInt(500);
+ log.warn("{}: temporary conflict, retrying in {} ms ... (thread={}, retry={})", name, sleep, Thread.currentThread().getName(), retries);
try {
- Thread.sleep(retryInterval);
+ Thread.sleep(sleep);
} catch (InterruptedException e) {}
retries++;
- execute(connection, command);
+ T result = execute(connection, command);
retries--;
+
+ return result;
} else {
- log.error("{}: temporary conflict could not be solved!", name);
+ log.error("{}: temporary conflict could not be solved! (error: {})", name, ex.getMessage());
- System.err.println("main exception:");
- ex.printStackTrace();
- System.err.println("next exception:");
- ex.getNextException().printStackTrace();
+ log.debug("main exception:",ex);
+ log.debug("next exception:",ex.getNextException());
throw ex;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
index 130e3e7..092cfcb 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
@@ -60,8 +60,8 @@ store.tliteral = INSERT INTO nodes (id,ntype,svalue,tvalue,ltype,createdAt
store.namespace = INSERT INTO namespaces (id,prefix,uri,createdAt) VALUES (?,?,?,?)
-store.triple = INSERT INTO triples (id,subject,predicate,object,context,inferred,createdAt) VALUES (?,?,?,?,?,?,?)
-#store.triple = MERGE INTO triples (id,subject,predicate,object,context,inferred,createdAt) KEY(id) VALUES (?,?,?,?,?,?,?)
+#store.triple = INSERT INTO triples (id,subject,predicate,object,context,inferred,createdAt) VALUES (?,?,?,?,?,?,?)
+store.triple = MERGE INTO triples (id,subject,predicate,object,context,inferred,createdAt) KEY(id) VALUES (?,?,?,?,?,?,?)
load.triple = SELECT id FROM triples WHERE subject = ? AND predicate = ? AND object = ? AND context = ? AND deleted = false AND inferred = true
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c7563c0d/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
index cbdbee4..487bfde 100644
--- a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
+++ b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/sesame/KiWiLocaleTest.java
@@ -30,6 +30,8 @@ import org.openrdf.model.Literal;
import org.openrdf.sail.SailConnection;
import org.openrdf.sail.SailException;
+import java.sql.SQLException;
+
/**
* Tests for testing locales against the KiWi Triple store
* (and its implementation of the ValueFactory)
@@ -55,7 +57,8 @@ public class KiWiLocaleTest {
}
@After
- public void shutdown() throws SailException {
+ public void shutdown() throws SailException, SQLException {
+ store.getPersistence().dropDatabase();
store.shutDown();
store = null;
vf = null;