You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/11/15 21:18:27 UTC
[1/2] git commit: bigger batch size for postgres, proper logging
Updated Branches:
refs/heads/develop 1d9325dc9 -> 7d848bdc7
bigger batch size for postgres, proper logging
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/869f2aeb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/869f2aeb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/869f2aeb
Branch: refs/heads/develop
Commit: 869f2aebc9c68c93dc41de6c14ce9bdcf05547a9
Parents: 1d9325d
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Fri Nov 15 21:15:16 2013 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Fri Nov 15 21:15:16 2013 +0100
----------------------------------------------------------------------
.../main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/869f2aeb/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
index 0972971..430e11d 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
@@ -341,7 +341,7 @@ public class KiWiLoader {
KiWiHandler handler;
if(kiwi.getDialect() instanceof PostgreSQLDialect) {
- config.setCommitBatchSize(10000);
+ config.setCommitBatchSize(100000);
handler = new KiWiPostgresHandler(store,config);
} else {
handler = new KiWiHandler(store,config);
@@ -355,7 +355,7 @@ public class KiWiLoader {
parser.parse(inStream,baseUri);
} catch (RDFHandlerException e) {
- log.error("RepositoryException: {}", e.getMessage());
+ log.error("RepositoryException:", e);
}
}
[2/2] git commit: retry on failed connection flush
Posted by ss...@apache.org.
retry on failed connection flush
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/7d848bdc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/7d848bdc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/7d848bdc
Branch: refs/heads/develop
Commit: 7d848bdc7a62cf785a606dc19bb5649e2257fa78
Parents: 869f2ae
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Fri Nov 15 21:18:22 2013 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Fri Nov 15 21:18:22 2013 +0100
----------------------------------------------------------------------
.../marmotta/kiwi/loader/pgsql/KiWiPostgresHandler.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/7d848bdc/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/KiWiPostgresHandler.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/KiWiPostgresHandler.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/KiWiPostgresHandler.java
index 3e6b0eb..95fe083 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/KiWiPostgresHandler.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/KiWiPostgresHandler.java
@@ -145,8 +145,16 @@ public class KiWiPostgresHandler extends KiWiHandler implements RDFHandler {
count++;
if(count % config.getCommitBatchSize() == 0) {
- flushBacklog();
- connection.commit();
+ try {
+ flushBacklog();
+ connection.commit();
+ } catch (SQLException ex) {
+ log.warn("could not flush out data ({}), retrying with fresh connection", ex.getCause().getMessage());
+ connection.close();
+ connection = store.getPersistence().getConnection();
+ flushBacklog();
+ connection.commit();
+ }
log.info("imported {} triples ({}/sec)", count, (config.getCommitBatchSize() * 1000) / (System.currentTimeMillis() - previous));
previous = System.currentTimeMillis();