You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by wi...@apache.org on 2015/10/12 18:06:21 UTC
marmotta git commit: MARMOTTA-584: adapted the kiwi loader test suite
Repository: marmotta
Updated Branches:
refs/heads/MARMOTTA-584 87a037ddc -> fa32ed2a7
MARMOTTA-584: adapted the kiwi loader test suite
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/fa32ed2a
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/fa32ed2a
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/fa32ed2a
Branch: refs/heads/MARMOTTA-584
Commit: fa32ed2a7180e07c8d8ad0c8a048ce09163453d9
Parents: 87a037d
Author: Sergio Fernández <wi...@apache.org>
Authored: Mon Oct 12 16:46:14 2015 +0200
Committer: Sergio Fernández <wi...@apache.org>
Committed: Mon Oct 12 16:46:14 2015 +0200
----------------------------------------------------------------------
.../marmotta/kiwi/loader/pgsql/PGCopyUtil.java | 34 ++++++++++----------
.../marmotta/kiwi/loader/PGCopyUtilTest.java | 5 ---
2 files changed, 17 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/marmotta/blob/fa32ed2a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/PGCopyUtil.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/PGCopyUtil.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/PGCopyUtil.java
index d6a8387..6483192 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/PGCopyUtil.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/PGCopyUtil.java
@@ -43,7 +43,7 @@ import java.util.List;
import java.util.Locale;
/**
- * Add file description here!
+ * Postgres copy utility
*
* @author Sebastian Schaffert (sschaffert@apache.org)
*/
@@ -51,14 +51,13 @@ public class PGCopyUtil {
private static Logger log = LoggerFactory.getLogger(PGCopyUtil.class);
-
final static CellProcessor[] nodeProcessors = new CellProcessor[] {
- new NotNull(), // node ID
+ new NotNull(), // node ID
new NodeTypeProcessor(), // ntype
new NotNull(), // svalue
new Optional(), // dvalue
new Optional(), // ivalue
- new SQLDateTimeProcessor(), // tvalue
+ new SQLDateTimeProcessor(), // tvalue
new Optional(), // tzoffset
new Optional(new SQLBooleanProcessor()), // bvalue
new Optional(new NodeIDProcessor()), // ltype
@@ -68,7 +67,7 @@ public class PGCopyUtil {
final static CellProcessor[] tripleProcessors = new CellProcessor[] {
- new NotNull(), // triple ID
+ new NotNull(), // triple ID
new NodeIDProcessor(), // subject
new NodeIDProcessor(), // predicate
new NodeIDProcessor(), // object
@@ -80,7 +79,6 @@ public class PGCopyUtil {
new SQLTimestampProcessor(), // deletedAt
};
-
// PostgreSQL expects the empty string to be quoted to distinguish between null and empty
final static CsvPreference nodesPreference = new CsvPreference.Builder('"', ',', "\r\n").useEncoder(new DefaultCsvEncoder() {
/**
@@ -135,34 +133,32 @@ public class PGCopyUtil {
writer.close();
}
-
-
public static void flushNodes(Iterable<KiWiNode> nodeBacklog, OutputStream out) throws IOException {
CsvListWriter writer = new CsvListWriter(new OutputStreamWriter(out), nodesPreference);
// reuse the same array to avoid unnecessary object allocation
- Object[] rowArray = new Object[11];
+ Object[] rowArray = new Object[12]; //FIXME: 11 in schema v4, 12 in v5
List<Object> row = Arrays.asList(rowArray);
for(KiWiNode n : nodeBacklog) {
if(n instanceof KiWiUriResource) {
KiWiUriResource u = (KiWiUriResource)n;
- createNodeList(rowArray, u.getId(), u.getClass(), u.stringValue(), null, null, null, null, null, null, null, u.getCreated());
+ createNodeList(rowArray, u.getId(), u.getClass(), u.stringValue(), null, null, null, null, null, null, null, u.getCreated(), null);
} else if(n instanceof KiWiAnonResource) {
KiWiAnonResource a = (KiWiAnonResource)n;
- createNodeList(rowArray, a.getId(), a.getClass(), a.stringValue(), null, null, null, null, null, null, null, a.getCreated());
+ createNodeList(rowArray, a.getId(), a.getClass(), a.stringValue(), null, null, null, null, null, null, null, a.getCreated(), null);
} else if(n instanceof KiWiIntLiteral) {
KiWiIntLiteral l = (KiWiIntLiteral)n;
- createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), l.getDoubleContent(), l.getIntContent(), null, null, null, l.getDatatype(), l.getLocale(), l.getCreated());
+ createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), l.getDoubleContent(), l.getIntContent(), null, null, null, l.getDatatype(), l.getLocale(), l.getCreated(), null);
} else if(n instanceof KiWiDoubleLiteral) {
KiWiDoubleLiteral l = (KiWiDoubleLiteral)n;
- createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), l.getDoubleContent(), null, null, null, null, l.getDatatype(), l.getLocale(), l.getCreated());
+ createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), l.getDoubleContent(), null, null, null, null, l.getDatatype(), l.getLocale(), l.getCreated(), null);
} else if(n instanceof KiWiBooleanLiteral) {
KiWiBooleanLiteral l = (KiWiBooleanLiteral)n;
- createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), null, null, null, null, l.booleanValue(), l.getDatatype(), l.getLocale(), l.getCreated());
+ createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), null, null, null, null, l.booleanValue(), l.getDatatype(), l.getLocale(), l.getCreated(), null);
} else if(n instanceof KiWiDateLiteral) {
KiWiDateLiteral l = (KiWiDateLiteral)n;
- createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), null, null, l.getDateContent(), l.getDateContent().getZone().getOffset(l.getDateContent()) / 1000, null, l.getDatatype(), l.getLocale(), l.getCreated());
+ createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), null, null, l.getDateContent(), l.getDateContent().getZone().getOffset(l.getDateContent()) / 1000, null, l.getDatatype(), l.getLocale(), l.getCreated(), null);
} else if(n instanceof KiWiStringLiteral) {
KiWiStringLiteral l = (KiWiStringLiteral)n;
@@ -176,7 +172,9 @@ public class PGCopyUtil {
// ignore, keep NaN
}
}
- createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), dbl_value, lng_value, null, null, null, l.getDatatype(), l.getLocale(), l.getCreated());
+ createNodeList(rowArray, l.getId(), l.getClass(), l.getContent(), dbl_value, lng_value, null, null, null, l.getDatatype(), l.getLocale(), l.getCreated(), null);
+ } else if(n instanceof KiWiGeometryLiteral) {
+ log.warn("geometries are not yet supported on bulk imports");
} else {
log.warn("unknown node type, cannot flush to import stream: {}", n.getClass());
}
@@ -186,7 +184,7 @@ public class PGCopyUtil {
writer.close();
}
- private static void createNodeList(Object[] a, Long id, Class type, String content, Double dbl, Long lng, DateTime date, Integer tzoffset, Boolean bool, URI dtype, Locale lang, Date created) {
+ private static void createNodeList(Object[] a, Long id, Class type, String content, Double dbl, Long lng, DateTime date, Integer tzoffset, Boolean bool, URI dtype, Locale lang, Date created, String geom) {
a[0] = id;
a[1] = type;
a[2] = content;
@@ -198,5 +196,7 @@ public class PGCopyUtil {
a[8] = dtype;
a[9] = lang != null ? lang.getLanguage() : "";
a[10] = created;
+ a[11] = geom; //FIXME: drop in v4 schema testing
}
+
}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/fa32ed2a/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java b/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java
index 37607e6..e7f27a9 100644
--- a/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java
+++ b/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java
@@ -102,9 +102,6 @@ public class PGCopyUtilTest {
}
}
-
-
-
@Test
public void testWriteNodes() throws IOException, SQLException {
KiWiConnection con = store.getPersistence().getConnection();
@@ -148,8 +145,6 @@ public class PGCopyUtilTest {
log.info("checked {} nodes in {} ms", nodes.size(), System.currentTimeMillis()-imported);
}
-
-
/**
* Return a random URI, with a 10% chance of returning a URI that has already been used.
* @return