You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/08/02 17:43:35 UTC
[1/2] git commit: some further improvements for concurrency,
hope now things are solved
Updated Branches:
refs/heads/develop 44c5d3bae -> d3757e494
some further improvements for concurrency, hope now things are solved
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/d3ce9754
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/d3ce9754
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/d3ce9754
Branch: refs/heads/develop
Commit: d3ce97549240cedee9d36adc553e8a08e267694f
Parents: 44c5d3b
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Fri Aug 2 17:43:05 2013 +0200
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Fri Aug 2 17:43:05 2013 +0200
----------------------------------------------------------------------
.../marmotta/commons/locking/StringLocks.java | 2 +-
.../marmotta/kiwi/config/KiWiConfiguration.java | 19 ++++++++++++++++++
.../kiwi/persistence/KiWiConnection.java | 21 +++++++++++++++-----
.../kiwi/persistence/KiWiPersistence.java | 16 +++++++++++++++
.../kiwi/persistence/h2/statements.properties | 8 ++++----
.../persistence/mysql/statements.properties | 8 ++++----
.../persistence/pgsql/statements.properties | 8 ++++----
7 files changed, 64 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/d3ce9754/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/locking/StringLocks.java
----------------------------------------------------------------------
diff --git a/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/locking/StringLocks.java b/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/locking/StringLocks.java
index 3b24aef..9a43f83 100644
--- a/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/locking/StringLocks.java
+++ b/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/locking/StringLocks.java
@@ -34,7 +34,7 @@ public class StringLocks {
private LoadingCache<String,Monitor> stringLocks;
public StringLocks() {
- stringLocks = CacheBuilder.newBuilder().build(new LockCacheLoader());
+ stringLocks = CacheBuilder.newBuilder().weakKeys().build(new LockCacheLoader());
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/d3ce9754/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
index a8443d2..ba53de9 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
@@ -88,6 +88,10 @@ public class KiWiConfiguration {
*/
private boolean memorySequences = true;
+
+ private boolean commitSequencesOnCommit = false;
+
+
public KiWiConfiguration(String name, String jdbcUrl, String dbUser, String dbPassword, KiWiDialect dialect) {
this(name, jdbcUrl, dbUser, dbPassword, dialect, null, null);
}
@@ -210,4 +214,19 @@ public class KiWiConfiguration {
public void setMemorySequences(boolean memorySequences) {
this.memorySequences = memorySequences;
}
+
+
+ public boolean isCommitSequencesOnCommit() {
+ return commitSequencesOnCommit;
+ }
+
+ /**
+ * This flag determines whether memory sequences should be stored back into the database on every commit or only
+ * when the repository shuts down. Saving back on every commit is safer, but has less performance.
+ *
+ * @param commitSequencesOnCommit
+ */
+ public void setCommitSequencesOnCommit(boolean commitSequencesOnCommit) {
+ this.commitSequencesOnCommit = commitSequencesOnCommit;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/d3ce9754/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
index 6484094..52f46a2 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
@@ -1900,6 +1900,22 @@ public class KiWiConnection {
* @see #setAutoCommit
*/
public void commit() throws SQLException {
+ if(persistence.getConfiguration().isCommitSequencesOnCommit()) {
+ commitMemorySequences();
+ }
+
+ deletedStatementsLog.clear();
+
+ if(connection != null) {
+ connection.commit();
+ }
+ }
+
+ /**
+ * Store the values of all memory sequences back into the database. Should be called at least on repository shutdown
+ * but possibly even when a transaction commits.
+ */
+ public void commitMemorySequences() throws SQLException {
if(persistence.getMemorySequences() != null) {
requireJDBCConnection();
@@ -1925,11 +1941,6 @@ public class KiWiConnection {
flushBatch();
}
- deletedStatementsLog.clear();
-
- if(connection != null) {
- connection.commit();
- }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/d3ce9754/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
index 1f42d41..f487554 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
@@ -536,6 +536,22 @@ public class KiWiPersistence {
}
public void shutdown() {
+ if(!configuration.isCommitSequencesOnCommit()) {
+ log.info("storing in-memory sequences in database ...");
+ try {
+ KiWiConnection connection = getConnection();
+ try {
+ connection.commitMemorySequences();
+ connection.commit();
+ } finally {
+ connection.close();
+ }
+ } catch (SQLException e) {
+ log.error("could not store back values of in-memory sequences", e);
+ }
+ }
+
+
garbageCollector.shutdown();
cacheManager.shutdown();
connectionPool.close();
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/d3ce9754/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
index a027722..f064342 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties
@@ -36,10 +36,10 @@ load.literal_by_v = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,
load.literal_by_vl = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE svalue = ? AND lang = ?
load.literal_by_vt = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE svalue = ? AND ltype = ?
-load.literal_by_iv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE ivalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_dv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE dvalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_tv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE tvalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_bv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE bvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_iv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE ivalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_dv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE dvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_tv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE tvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_bv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE bvalue = ? AND lang IS NULL AND ltype = ?
load.namespace_prefix = SELECT id,prefix,uri,createdAt FROM namespaces WHERE prefix = ?;
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/d3ce9754/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties
index d52d5b2..af561df 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/mysql/statements.properties
@@ -44,10 +44,10 @@ load.literal_by_v = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,
load.literal_by_vl = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE svalue = ? AND lang = ?
load.literal_by_vt = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE svalue = ? AND ltype = ?
-load.literal_by_iv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE ivalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_dv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE dvalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_tv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE tvalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_bv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE bvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_iv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE ivalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_dv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE dvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_tv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE tvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_bv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE bvalue = ? AND lang IS NULL AND ltype = ?
load.namespace_prefix = SELECT id,prefix,uri,createdAt FROM namespaces WHERE prefix = ?;
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/d3ce9754/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties
index a38f3e0..9870b71 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/statements.properties
@@ -39,10 +39,10 @@ load.literal_by_v = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,
load.literal_by_vl = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE svalue = ? AND lang = ?
load.literal_by_vt = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE svalue = ? AND ltype = ?
-load.literal_by_iv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE ivalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_dv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE dvalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_tv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE tvalue = ? AND lang IS NULL AND ltype = ?
-load.literal_by_bv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE bvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_iv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE ivalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_dv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE dvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_tv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE tvalue = ? AND lang IS NULL AND ltype = ?
+load.literal_by_bv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE bvalue = ? AND lang IS NULL AND ltype = ?
load.namespace_prefix = SELECT id,prefix,uri,createdAt FROM namespaces WHERE prefix = ?;
load.namespace_uri = SELECT id,prefix,uri,createdAt FROM namespaces WHERE uri = ?;
[2/2] git commit: changed back number of runs for postgres
concurrency test
Posted by ss...@apache.org.
changed back number of runs for postgres concurrency test
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/d3757e49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/d3757e49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/d3757e49
Branch: refs/heads/develop
Commit: d3757e4944f7c131edc46785e9fa5044736c3f2c
Parents: d3ce975
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Fri Aug 2 17:43:28 2013 +0200
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Fri Aug 2 17:43:28 2013 +0200
----------------------------------------------------------------------
.../org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/d3757e49/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
index 70eedf2..b3c97c2 100644
--- a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
+++ b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PostgreSQLConcurrencyTest.java
@@ -138,8 +138,8 @@ public class PostgreSQLConcurrencyTest {
long tripleCount = 0;
@Test
- @Concurrent(count = 15)
- @Repeating(repetition = 10000)
+ @Concurrent(count = 10)
+ @Repeating(repetition = 10)
public void testConcurrency() throws Exception {
runs++;