You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/11/15 22:47:11 UTC
git commit: support empty string literals in bulk import
Updated Branches:
refs/heads/develop e4da0de6e -> 635cab616
support empty string literals in bulk import
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/635cab61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/635cab61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/635cab61
Branch: refs/heads/develop
Commit: 635cab616a360515d9c8acacd18c8a1211a6ddbe
Parents: e4da0de
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Fri Nov 15 22:47:07 2013 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Fri Nov 15 22:47:07 2013 +0100
----------------------------------------------------------------------
.../kiwi/loader/pgsql/csv/PGCopyUtil.java | 20 +++++++++++++++++++-
.../marmotta/kiwi/loader/PGCopyUtilTest.java | 14 ++++++++++++--
2 files changed, 31 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/635cab61/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/csv/PGCopyUtil.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/csv/PGCopyUtil.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/csv/PGCopyUtil.java
index 075ecb1..2e089ff 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/csv/PGCopyUtil.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/pgsql/csv/PGCopyUtil.java
@@ -17,8 +17,10 @@ import org.supercsv.cellprocessor.Optional;
import org.supercsv.cellprocessor.constraint.NotNull;
import org.supercsv.cellprocessor.constraint.Unique;
import org.supercsv.cellprocessor.ift.CellProcessor;
+import org.supercsv.encoder.DefaultCsvEncoder;
import org.supercsv.io.CsvListWriter;
import org.supercsv.prefs.CsvPreference;
+import org.supercsv.util.CsvContext;
import java.io.IOException;
import java.io.OutputStream;
@@ -68,6 +70,22 @@ public class PGCopyUtil {
};
+ // PostgreSQL expects the empty string to be quoted to distinguish between null and empty
+ final static CsvPreference nodesPreference = new CsvPreference.Builder('"', ',', "\r\n").useEncoder(new DefaultCsvEncoder() {
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String encode(String input, CsvContext context, CsvPreference preference) {
+ if("".equals(input)) {
+ return "\"\"";
+ } else {
+ return super.encode(input, context, preference);
+ }
+ }
+ }).build();
+
+
/**
* Return a PGConnection wrapped by the tomcat connection pool so we are able to access PostgreSQL specific functionality.
* @param con
@@ -99,7 +117,7 @@ public class PGCopyUtil {
public static void flushNodes(Iterable<KiWiNode> nodeBacklog, OutputStream out) throws IOException {
- CsvListWriter writer = new CsvListWriter(new OutputStreamWriter(out), CsvPreference.STANDARD_PREFERENCE);
+ CsvListWriter writer = new CsvListWriter(new OutputStreamWriter(out), nodesPreference);
for(KiWiNode n : nodeBacklog) {
List<Object> row = null;
if(n instanceof KiWiUriResource) {
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/635cab61/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java b/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java
index c264f45..c4df285 100644
--- a/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java
+++ b/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/PGCopyUtilTest.java
@@ -58,6 +58,7 @@ public class PGCopyUtilTest {
final static KiWiUriResource TYPE_DBL = createURI(XSD.Double.stringValue());
final static KiWiUriResource TYPE_BOOL = createURI(XSD.Boolean.stringValue());
final static KiWiUriResource TYPE_DATE = createURI(XSD.DateTime.stringValue());
+ final static KiWiStringLiteral EMPTY = createLiteral("");
@@ -107,9 +108,10 @@ public class PGCopyUtilTest {
nodes.add(TYPE_DBL);
nodes.add(TYPE_BOOL);
nodes.add(TYPE_DATE);
+ nodes.add(EMPTY);
// randomly create 10000 nodes
- for(int i=0; i<100000; i++) {
+ for(int i=0; i<10000; i++) {
nodes.add(randomObject());
}
@@ -125,10 +127,11 @@ public class PGCopyUtilTest {
// check if database contains the nodes (based on ID)
PreparedStatement stmt = con.getJDBCConnection().prepareStatement("SELECT * FROM nodes WHERE id = ?");
- for(int i=0; i<id; i++) {
+ for(int i=0; i<nodes.size(); i++) {
stmt.setLong(1, (long)i);
ResultSet dbResult = stmt.executeQuery();
Assert.assertTrue(dbResult.next());
+ Assert.assertEquals(nodes.get(i).stringValue(),dbResult.getString("svalue"));
}
log.info("checked {} nodes in {} ms", nodes.size(), System.currentTimeMillis()-imported);
@@ -182,4 +185,11 @@ public class PGCopyUtilTest {
return object;
}
+ protected static KiWiStringLiteral createLiteral(String data) {
+ KiWiStringLiteral r = new KiWiStringLiteral(data);
+ r.setId(id++);
+ return r;
+ }
+
+
}