You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/07/22 16:36:48 UTC
[1/2] git commit: introduced some automatic consistency fixing
Updated Branches:
refs/heads/develop 16fbcbb16 -> 3c4ccff0d
introduced some automatic consistency fixing
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/587f1387
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/587f1387
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/587f1387
Branch: refs/heads/develop
Commit: 587f1387a0184b6a0b76ea14dfe60160769232aa
Parents: f62c710
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Mon Jul 22 16:36:21 2013 +0200
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Mon Jul 22 16:36:21 2013 +0200
----------------------------------------------------------------------
.../kiwi/persistence/KiWiGarbageCollector.java | 90 ++++++++++++++++++--
.../kiwi/persistence/h2/create_base_tables.sql | 3 +-
.../persistence/pgsql/create_base_tables.sql | 3 +-
.../marmotta/kiwi/test/PersistenceTest.java | 84 ++++++++++++++++++
4 files changed, 168 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/587f1387/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
index f769e37..4ba05db 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiGarbageCollector.java
@@ -24,6 +24,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
@@ -102,26 +103,99 @@ public class KiWiGarbageCollector extends Thread {
protected boolean checkConsistency() throws SQLException {
boolean consistent = true;
- String checkNodeDuplicatesQuery = "SELECT svalue, count(id) FROM nodes WHERE ntype='uri' group by svalue having count(id) > 1";
+ String checkNodeDuplicatesQuery = "SELECT svalue, ntype, count(id) FROM nodes group by svalue, ntype having count(id) > 1";
try(Connection con = persistence.getJDBCConnection()) {
PreparedStatement checkNodeDuplicatesStatement = con.prepareStatement(checkNodeDuplicatesQuery);
- ResultSet result = checkNodeDuplicatesStatement.executeQuery();
- if(result.next()) {
- log.warn("DATABASE INCONSISTENCY: duplicate node entries found, manual repair required!");
- do {
- log.warn(" - inconsistent resource: {}", result.getString("svalue"));
- } while(result.next());
+ try(ResultSet result = checkNodeDuplicatesStatement.executeQuery()) {
+ if(result.next()) {
+ log.warn("DATABASE INCONSISTENCY: duplicate node entries found, please try to fix the consistency with fixConsistency()!");
+ do {
+ if(result.getString("ntype").equals("uri")) {
+ log.warn(" - inconsistent resource: {}", result.getString("svalue"));
+ }
+ } while(result.next());
- consistent = false;
+ consistent = false;
+ }
}
}
+ if(!consistent) {
+ log.warn("DATABASE INCONSISTENCY: attempting to auto-fix inconsistencies where possible");
+ try {
+ fixConsistency();
+ } catch (SQLException ex) {
+ log.error("DATABASE INCONSISTENCY: auto-fixing inconsistencies failed ({})", ex.getMessage());
+ }
+ }
+
return consistent;
}
+
+ protected void fixConsistency() throws SQLException {
+ String checkNodeDuplicatesQuery = "SELECT svalue, ntype, count(id), max(id) FROM nodes group by svalue, ntype having count(id) > 1";
+ String getNodeIdsQuery = "SELECT id FROM nodes WHERE svalue = ? AND ntype = ? AND id != ?";
+
+ try(Connection con = persistence.getJDBCConnection(true)) {
+ PreparedStatement checkNodeDuplicatesStatement = con.prepareStatement(checkNodeDuplicatesQuery);
+ PreparedStatement getNodeIdsStatement = con.prepareStatement(getNodeIdsQuery);
+
+ ResultSet result = checkNodeDuplicatesStatement.executeQuery();
+ while(result.next()) {
+ if(result.getString("ntype").equals("uri")) {
+ log.warn("DATABASE INCONSISTENCY: attempting to fix references for resource {}", result.getString("svalue"));
+ } else {
+ log.warn("DATABASE INCONSISTENCY: attempting to fix references for literal or anonymous node");
+ }
+
+ long latest_id = result.getLong(4);
+
+ // first we collect all ids of nodes that have the same svalue and ntype
+ getNodeIdsStatement.clearParameters();
+ getNodeIdsStatement.setString(1, result.getString("svalue"));
+ getNodeIdsStatement.setString(2,result.getString("ntype"));
+ getNodeIdsStatement.setLong(3, latest_id);
+
+ ArrayList<Long> ids = new ArrayList<>();
+ try(ResultSet idResult = getNodeIdsStatement.executeQuery()) {
+ while(idResult.next()) {
+ ids.add(idResult.getLong(1));
+ }
+ }
+ getNodeIdsStatement.close();
+
+ // then we "fix" the triples table by making sure that all subjects, predicates, objects and contexts point to
+ // the latest version only; we use the nodes dependency table for this purpose
+ for(TableDependency dep : nodeTableDependencies) {
+ String fixNodeIdsQuery = "UPDATE " + dep.table + " SET " + dep.column + " = " + latest_id + " WHERE " + dep.column + " = ?";
+ PreparedStatement fixNodeIdsStatement = con.prepareStatement(fixNodeIdsQuery);
+ for(Long id : ids) {
+ fixNodeIdsStatement.setLong(1, id);
+ fixNodeIdsStatement.addBatch();
+ }
+ fixNodeIdsStatement.executeBatch();
+ fixNodeIdsStatement.close();
+ }
+
+ // finally we clean up all now unused node ids
+ String deleteDuplicatesQuery = "DELETE FROM nodes WHERE id = ?";
+ PreparedStatement deleteDuplicatesStatement = con.prepareStatement(deleteDuplicatesQuery);
+ for(Long id : ids) {
+ deleteDuplicatesStatement.setLong(1, id);
+ deleteDuplicatesStatement.addBatch();
+ }
+ deleteDuplicatesStatement.executeBatch();
+ deleteDuplicatesStatement.close();
+ }
+ checkNodeDuplicatesStatement.close();
+ }
+ }
+
+
protected int garbageCollect() throws SQLException {
round++;
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/587f1387/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
index 6769ca2..f6e1722 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql
@@ -28,8 +28,7 @@ CREATE TABLE nodes (
ltype bigint REFERENCES nodes(id),
lang varchar(5),
createdAt timestamp NOT NULL DEFAULT now(),
- PRIMARY KEY(id),
- UNIQUE(ntype,svalue)
+ PRIMARY KEY(id)
);
CREATE TABLE triples (
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/587f1387/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
index 7f5f8c4..f165c68 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/pgsql/create_base_tables.sql
@@ -29,8 +29,7 @@ CREATE TABLE nodes (
ltype bigint REFERENCES nodes(id),
lang varchar(5),
createdAt timestamp NOT NULL DEFAULT now(),
- PRIMARY KEY(id),
- UNIQUE(ntype,svalue)
+ PRIMARY KEY(id)
);
CREATE TABLE triples (
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/587f1387/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java
index 52ca670..a51b701 100644
--- a/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java
+++ b/libraries/kiwi/kiwi-triplestore/src/test/java/org/apache/marmotta/kiwi/test/PersistenceTest.java
@@ -585,6 +585,89 @@ public class PersistenceTest {
* @throws SQLException
*/
@Test
+ public void testStoreBigStringLiteral() throws SQLException {
+ KiWiConnection connection = persistence.getConnection();
+ try {
+ KiWiUriResource uri = new KiWiUriResource("http://localhost/"+ RandomStringUtils.randomAlphanumeric(8));
+
+ // add a new URI to the triple store and check if it exists afterwards, before and after commit
+ KiWiStringLiteral literal = new KiWiStringLiteral(RandomStringUtils.randomAlphanumeric(16384), null, uri);
+ connection.storeNode(literal, false);
+
+ // check if it then has a database ID
+ Assert.assertNotNull(literal.getId());
+
+ KiWiNode testLiteral1 = connection.loadNodeById(literal.getId());
+
+ // needs to be equal, and should also be the identical object!
+ Assert.assertEquals(literal,testLiteral1);
+ Assert.assertEquals(uri,((KiWiLiteral)testLiteral1).getType());
+ Assert.assertTrue(literal == testLiteral1);
+
+ connection.commit();
+
+ KiWiNode testLiteral2 = connection.loadNodeById(literal.getId());
+
+ // needs to be equal, and should also be the identical object!
+ Assert.assertEquals(literal,testLiteral2);
+ Assert.assertEquals(uri,((KiWiLiteral)testLiteral2).getType());
+ Assert.assertTrue(literal == testLiteral2);
+
+ KiWiNode testLiteral3 = connection.loadLiteral(literal.stringValue(),null,uri);
+
+ // needs to be equal, and should also be the identical object!
+ Assert.assertEquals(literal,testLiteral3);
+ Assert.assertEquals(uri,((KiWiLiteral)testLiteral3).getType());
+ Assert.assertTrue(literal == testLiteral3);
+
+ connection.commit();
+
+
+ // clear cache and test again
+ persistence.clearCache();
+ KiWiNode testLiteral4 = connection.loadNodeById(literal.getId());
+
+ // needs to be equal, but now it should not be the same object!
+ Assert.assertEquals(literal,testLiteral4);
+ Assert.assertEquals(uri,((KiWiLiteral)testLiteral4).getType());
+ Assert.assertTrue(literal != testLiteral4);
+
+ KiWiNode testLiteral5 = connection.loadLiteral(literal.stringValue(),null,uri);
+
+ // needs to be equal, but now it should not be the same object!
+ Assert.assertEquals(literal,testLiteral5);
+ Assert.assertEquals(uri,((KiWiLiteral)testLiteral5).getType());
+ Assert.assertTrue(literal != testLiteral5);
+
+ connection.commit();
+
+ // finally do a test on the nodes table, it should contain exactly one entry
+ PreparedStatement checkNodeStmt = connection.getJDBCConnection().prepareStatement("SELECT * FROM nodes WHERE ntype='string'");
+ ResultSet result = checkNodeStmt.executeQuery();
+
+ Assert.assertTrue(result.next());
+ Assert.assertEquals((long)literal.getId(),result.getLong("id"));
+ Assert.assertEquals(literal.stringValue(),result.getString("svalue"));
+ Assert.assertEquals("string",result.getString("ntype"));
+ Assert.assertNull(result.getString("lang"));
+ Assert.assertEquals((long) uri.getId(), result.getLong("ltype"));
+
+ result.close();
+ connection.commit();
+ } finally {
+ connection.close();
+ }
+
+ }
+
+
+
+ /**
+ * Test storing and loading string literals (with type and without language).
+ *
+ * @throws SQLException
+ */
+ @Test
public void testStoreIntLiteral() throws SQLException {
KiWiConnection connection = persistence.getConnection();
try {
@@ -997,6 +1080,7 @@ public class PersistenceTest {
}
+
/**
* Test storing, querying and deleting triples
*/
[2/2] git commit: Merge remote-tracking branch 'origin/develop' into
develop
Posted by ss...@apache.org.
Merge remote-tracking branch 'origin/develop' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/3c4ccff0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/3c4ccff0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/3c4ccff0
Branch: refs/heads/develop
Commit: 3c4ccff0d01e6f798332f453583bb8f4a2ed179b
Parents: 587f138 16fbcbb
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Mon Jul 22 16:36:36 2013 +0200
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Mon Jul 22 16:36:36 2013 +0200
----------------------------------------------------------------------
platform/marmotta-core/src/main/resources/templates/404.ftl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------