You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ja...@apache.org on 2013/10/03 17:14:20 UTC
[1/2] git commit: MARMOTTA-310 continued implementation
Updated Branches:
refs/heads/MARMOTTA-310 f97403951 -> 188d7533f
MARMOTTA-310 continued implementation
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/780948f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/780948f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/780948f8
Branch: refs/heads/MARMOTTA-310
Commit: 780948f8cdcf04d72504b58d9888cdf06294c352
Parents: f974039
Author: Jakob Frank <ja...@apache.org>
Authored: Wed Oct 2 17:18:59 2013 +0200
Committer: Jakob Frank <ja...@apache.org>
Committed: Wed Oct 2 17:18:59 2013 +0200
----------------------------------------------------------------------
.../apache/marmotta/kiwi/loader/KiWiLoader.java | 93 +++++++++++++++-----
1 file changed, 71 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/780948f8/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
index 9555b6c..a93b21e 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
@@ -34,17 +34,15 @@ import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.marmotta.kiwi.config.KiWiConfiguration;
import org.apache.marmotta.kiwi.persistence.KiWiDialect;
-import org.apache.marmotta.kiwi.persistence.KiWiPersistence;
import org.apache.marmotta.kiwi.persistence.h2.H2Dialect;
import org.apache.marmotta.kiwi.persistence.mysql.MySQLDialect;
import org.apache.marmotta.kiwi.persistence.pgsql.PostgreSQLDialect;
-import org.apache.marmotta.kiwi.reasoner.sail.KiWiReasoningSail;
import org.apache.marmotta.kiwi.sail.KiWiStore;
import org.apache.marmotta.kiwi.transactions.api.TransactionalSail;
import org.apache.marmotta.kiwi.transactions.sail.KiWiTransactionalSail;
-import org.apache.marmotta.kiwi.versioning.sail.KiWiVersioningSail;
import org.openrdf.repository.RepositoryException;
import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.repository.sail.SailRepositoryConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,28 +78,54 @@ public class KiWiLoader {
CommandLineParser parser = new PosixParser();
try {
CommandLine cmd = parser.parse(options, args);
-
- final String dbCon, dbUser, dbPasswd;
- dbCon = cmd.getOptionValue('d');
- dbUser = cmd.getOptionValue('U');
-
- final KiWiDialect dialect = getDialect(cmd, dbCon);
- dbPasswd = getPassword(cmd, dbCon, dbUser);
-
- KiWiConfiguration kiwi = new KiWiConfiguration("kiwiLoader", dbCon, dbUser, dbPasswd, dialect);
-
String baseUri = endWith("http://localhost/", "/"),
context = baseUri+"context/default",
format = null;
boolean gzip = false;
+ String dbCon = null, dbUser = null, dbPasswd = null;
+ KiWiDialect dialect = null;
+
if (cmd.hasOption('c')) {
Configuration conf = new PropertiesConfiguration(cmd.getOptionValue('c'));
baseUri = endWith(conf.getString("kiwi.context", baseUri), "/");
context = conf.getString("kiwi.context", baseUri) + "context/default";
+
+ dbCon = conf.getString("database.url");
+ dbUser = conf.getString("database.user");
+ dbPasswd = conf.getString("database.password");
+
+ String dbType = conf.getString("database.type");
+ if ("h2".equalsIgnoreCase(dbType)) {
+ dialect = new H2Dialect();
+ } else if ("mysql".equalsIgnoreCase(dbType)) {
+ dialect = new MySQLDialect();
+ } else if ("postgres".equalsIgnoreCase(dbType)) {
+ dialect = new PostgreSQLDialect();
+ }
+ }
+
+ if (cmd.hasOption('d')) {
+ dbCon = cmd.getOptionValue('d');
+ }
+ if (cmd.hasOption('U')) {
+ dbUser = cmd.getOptionValue('U');
}
+ dialect = getDialect(cmd, dbCon, dialect);
+ dbPasswd = getPassword(cmd, dbCon, dbUser, dbPasswd);
+
+ if (dbCon == null || dbUser == null || dbPasswd == null || dialect == null) {
+ log.error("Cannot import without database connection!\n"
+ +"Please provide the database configuration either\n"
+ +"- in a kiwi-config file (-c), and/or\n"
+ +"- as commandline parameters (-d, -U, -P, -D");
+ System.exit(3);
+ }
+
+ KiWiConfiguration kiwi = new KiWiConfiguration("kiwiLoader", dbCon, dbUser, dbPasswd, dialect);
+
if (cmd.hasOption('b')) {
baseUri = endWith(cmd.getOptionValue('b', baseUri), "/");
context = cmd.getOptionValue('b', baseUri) + "context/default";
@@ -124,24 +148,35 @@ public class KiWiLoader {
throw new ParseException("No files to import");
}
+ log.info("Initializing KiWiLoader for {}; user: {}, password: {}", dbCon, dbUser, String.format("%"+dbPasswd.length()+"s", "*"));
KiWiLoader loader = new KiWiLoader(kiwi, baseUri, context);
loader.setVersioningEnabled(cmd.hasOption("versioning"));
loader.setReasoningEnabled(cmd.hasOption("reasoning"));
loader.initialize();
+ log.info("Starting import");
for (String inFile: inputFiles) {
+ log.info("Importing {}", inFile);
+ long start = System.currentTimeMillis();
loader.load(inFile, format, gzip);
+ long dur = System.currentTimeMillis() - start;
+ log.info("Import completed ({}): {}", inFile, dur);
}
+ log.info("Import complete, shutting down.");
loader.shutdown();
+ log.info("Exiting");
} catch (ParseException e) {
log.error(e.getMessage());
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp(KiWiLoader.class.getSimpleName(), options, true);
+ System.exit(1);
} catch (ConfigurationException e) {
log.error("Could not read system-config.properties: {}", e.getMessage());
+ System.exit(1);
} catch (RepositoryException e) {
log.error("Sesame-Exception: {}", e.getMessage(), e);
+ System.exit(2);
}
}
@@ -162,24 +197,37 @@ public class KiWiLoader {
}
public void load(String file, String format, boolean gzip) {
- // TODO Auto-generated method stub
-
+ try {
+ final SailRepositoryConnection con = repository.getConnection();
+ try {
+ con.begin();
+ // TODO Auto-generated method stub
+ con.commit();
+ } finally {
+ con.close();
+ }
+ } catch (RepositoryException re) {
+ log.error("RepositoryException: {}", re.getMessage());
+ }
}
public synchronized void initialize() throws RepositoryException {
if (repository != null && repository.isInitialized()) {
throw new IllegalStateException("repository already initialized");
}
- KiWiStore store = new KiWiStore((kiwi));
+ log.debug("initializing kiwi-store: {}", kiwi);
+ KiWiStore store = new KiWiStore(kiwi);
TransactionalSail tSail = new KiWiTransactionalSail(store);
if (isVersioningEnabled) {
+ log.debug("enabling versioning...");
// TODO: Add Versioning
// tSail = new KiWiVersioningSail(tSail);
log.warn("versioning not yet supported/implemented");
}
if (isReasoningEnabled) {
+ log.debug("enabling reasoner...");
// TODO: Add Reasoning
// tSail = new KiWiReasoningSail(tSail, null);
log.warn("reasoning not yet supported/implemented");
@@ -189,7 +237,7 @@ public class KiWiLoader {
repository.initialize();
}
- private static String getPassword(CommandLine cmd, String jdbc, String user) throws ParseException {
+ private static String getPassword(CommandLine cmd, String jdbc, String user, String password) throws ParseException {
if (cmd.hasOption('P')) {
if (cmd.getOptionValue('P') != null) {
return cmd.getOptionValue('P');
@@ -203,15 +251,17 @@ public class KiWiLoader {
return new String(passwordArray);
}
} else {
- return "";
+ return password;
}
}
- private static KiWiDialect getDialect(CommandLine cmd, final String dbCon)
+ private static KiWiDialect getDialect(CommandLine cmd, String dbCon, KiWiDialect dialect)
throws ParseException {
// get the dialect
final String dbDialect = cmd.getOptionValue('D');
- if (dbDialect == null) {
+ if (!cmd.hasOption('D')) {
+ return dialect;
+ } else if (dbDialect == null) {
log.info("No dialect provded, trying to guess based on dbUrl: {}", dbCon);
// try guessing
if (dbCon.startsWith("jdbc:h2:")) {
@@ -251,6 +301,7 @@ public class KiWiLoader {
final Option context = new Option("C", "context", true, "context to import into");
context.setArgName("context");
context.setArgs(1);
+ context.setOptionalArg(false);
options.addOption(context);
options.addOption("z", false, "Input file is gzip compressed");
@@ -266,13 +317,11 @@ public class KiWiLoader {
options.addOption(input);
final Option dbUrl = new Option("d", "database", true, "jdbc connection string");
- dbUrl.setRequired(true);
dbUrl.setArgName("jdbc-url");
dbUrl.setArgs(1);
options.addOption(dbUrl);
final Option dbUser = new Option("U", "user", true, "database user");
- dbUser.setRequired(true);
dbUser.setArgName("dbUser");
options.addOption(dbUser);
[2/2] git commit: MARMOTTA-310: initial version of the KiWiLoader
ready, started with UnitTests.
Posted by ja...@apache.org.
MARMOTTA-310: initial version of the KiWiLoader ready, started with UnitTests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/188d7533
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/188d7533
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/188d7533
Branch: refs/heads/MARMOTTA-310
Commit: 188d7533fe6ea30fbb3716dd7b7f21276e4f330a
Parents: 780948f
Author: Jakob Frank <ja...@apache.org>
Authored: Thu Oct 3 17:13:27 2013 +0200
Committer: Jakob Frank <ja...@apache.org>
Committed: Thu Oct 3 17:13:27 2013 +0200
----------------------------------------------------------------------
libraries/kiwi/kiwi-loader/pom.xml | 13 +-
.../apache/marmotta/kiwi/loader/KiWiLoader.java | 309 ++++++++++++++-----
.../marmotta/kiwi/loader/KiWiLoaderTest.java | 64 ++++
3 files changed, 315 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/188d7533/libraries/kiwi/kiwi-loader/pom.xml
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/pom.xml b/libraries/kiwi/kiwi-loader/pom.xml
index a31aa47..c1e1470 100644
--- a/libraries/kiwi/kiwi-loader/pom.xml
+++ b/libraries/kiwi/kiwi-loader/pom.xml
@@ -26,7 +26,7 @@
<artifactId>kiwi-loader</artifactId>
<name>KiWi Triplestore: DB Loader</name>
- <description>Standalone module to load rdf files into a KiWi Database</description>
+ <description>Standalone module to load RDF files into a KiWi Database</description>
<build>
<plugins>
@@ -121,5 +121,16 @@
<artifactId>logback-classic</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.marmotta</groupId>
+ <artifactId>kiwi-triplestore</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.openrdf.sesame</groupId>
+ <artifactId>sesame-rio-turtle</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/188d7533/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
index a93b21e..e2fa012 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/KiWiLoader.java
@@ -18,9 +18,15 @@
package org.apache.marmotta.kiwi.loader;
import java.io.Console;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -40,13 +46,65 @@ import org.apache.marmotta.kiwi.persistence.pgsql.PostgreSQLDialect;
import org.apache.marmotta.kiwi.sail.KiWiStore;
import org.apache.marmotta.kiwi.transactions.api.TransactionalSail;
import org.apache.marmotta.kiwi.transactions.sail.KiWiTransactionalSail;
+import org.openrdf.model.Resource;
import org.openrdf.repository.RepositoryException;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.repository.sail.SailRepositoryConnection;
+import org.openrdf.rio.RDFFormat;
+import org.openrdf.rio.RDFParseException;
+import org.openrdf.sail.Sail;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
+ * {@link KiWiLoader} is a fastpath importer into a kiwi database. It is meant
+ * for importing large datafiles into a kiwi RDF store, avoiding the overhead of
+ * the full marmotta platform.
+ * <p>
+ * The usage is <br>
+ * {@code KiWiLoader
+ * [-h]
+ * [-c <config>]
+ * [-b <baseUri>]
+ * [-C <context>]
+ * [-d <jdbc-url>]
+ * [-D <dialect>]
+ * [-U <dbUser>]
+ * [-P [<passwd>]]
+ * [-f <mime-type>]
+ * [-z]
+ * [-i <rdf-file> ...]
+ * [--reasoning]
+ * [--versioning]
+ * <rdf-file> ...
+ * }
+ * <pre>
+ * General Settings:
+ * -h,--help print this help
+ *
+ * Base Settings:
+ * -c,--config <config> Marmotta system-config.properties file
+ * -b,--baseUri <baseUri> baseUri during the import (fallback:
+ * kiwi.context from the config file)
+ * -C,--context <context> context to import into
+ *
+ * Database:
+ * -d,--database <jdbc-url> jdbc connection string
+ * -D,--dbDialect <dialect> database dialect (h2, mysql, postgres)
+ * -U,--user <dbUser> database user
+ * -P,--dbPasswd <passwd> database password
+ *
+ * Import:
+ * -f,--format <mime-type> format of rdf file (if guessing based on the
+ * extension does not work)
+ * -z Input file is gzip compressed
+ * -i,--file <rdf-file> input file(s) to load
+ *
+ * KiWi Extra Settings:
+ * --reasoning enable reasoning
+ * --versioning enable versioning
+ * </pre>
+ *
* @author Jakob Frank <ja...@apache.org>
*
*/
@@ -59,18 +117,18 @@ public class KiWiLoader {
private boolean isVersioningEnabled;
private boolean isReasoningEnabled;
private SailRepository repository;
-
+
public KiWiLoader(KiWiConfiguration kiwi, String baseUri, String context) {
this.kiwi = kiwi;
this.baseUri = baseUri;
this.context = context;
-
+
isVersioningEnabled = false;
isReasoningEnabled = false;
}
/**
- * @param args
+ * @param args CLI params
*/
public static void main(String[] args) {
Options options = buildOptions();
@@ -79,23 +137,29 @@ public class KiWiLoader {
try {
CommandLine cmd = parser.parse(options, args);
+ if (cmd.hasOption("help")) {
+ printHelp(options);
+ return;
+ }
+
String baseUri = endWith("http://localhost/", "/"),
- context = baseUri+"context/default",
- format = null;
+ context = null, //baseUri+"context/default",
+ format = null;
boolean gzip = false;
-
+
String dbCon = null, dbUser = null, dbPasswd = null;
KiWiDialect dialect = null;
-
+
if (cmd.hasOption('c')) {
+ // load as much as possible from the config file
Configuration conf = new PropertiesConfiguration(cmd.getOptionValue('c'));
baseUri = endWith(conf.getString("kiwi.context", baseUri), "/");
- context = conf.getString("kiwi.context", baseUri) + "context/default";
-
+ //context = conf.getString("kiwi.context", baseUri) + "context/default";
+
dbCon = conf.getString("database.url");
dbUser = conf.getString("database.user");
dbPasswd = conf.getString("database.password");
-
+
String dbType = conf.getString("database.type");
if ("h2".equalsIgnoreCase(dbType)) {
dialect = new H2Dialect();
@@ -105,71 +169,112 @@ public class KiWiLoader {
dialect = new PostgreSQLDialect();
}
}
-
+
+ // database url
if (cmd.hasOption('d')) {
dbCon = cmd.getOptionValue('d');
}
+ // database user
if (cmd.hasOption('U')) {
dbUser = cmd.getOptionValue('U');
}
-
+
+ // database dialect, use config value as fallback if present.
dialect = getDialect(cmd, dbCon, dialect);
+ // database passsword; if -P is given but no password, ask the user to enter it.
dbPasswd = getPassword(cmd, dbCon, dbUser, dbPasswd);
-
+
+ // We need a db-configuration, otherwise this does not make any sense.
if (dbCon == null || dbUser == null || dbPasswd == null || dialect == null) {
log.error("Cannot import without database connection!\n"
+"Please provide the database configuration either\n"
+"- in a kiwi-config file (-c), and/or\n"
- +"- as commandline parameters (-d, -U, -P, -D");
+ +"- as commandline parameters (-d, -U, -P, -D)");
System.exit(3);
}
-
+
KiWiConfiguration kiwi = new KiWiConfiguration("kiwiLoader", dbCon, dbUser, dbPasswd, dialect);
-
+
+ // set the baseUri
if (cmd.hasOption('b')) {
baseUri = endWith(cmd.getOptionValue('b', baseUri), "/");
- context = cmd.getOptionValue('b', baseUri) + "context/default";
+ //context = cmd.getOptionValue('b', baseUri) + "context/default";
}
-
+
+ // the context to import into.
if (cmd.hasOption('C')) {
context = cmd.getOptionValue('C', context);
}
- format = cmd.getOptionValue('f');
+ // force uncompressing the files (will try to guess if this option is not set)
gzip = cmd.hasOption('z');
-
- ArrayList<String> inputFiles = new ArrayList<>();
+
+ // the format to use as fallback; will try to guess based on the filename.
+ format = cmd.getOptionValue('f');
+ final RDFFormat fmt;
+ if (format != null) {
+ fmt = RDFFormat.forMIMEType(format);
+ if (fmt == null) {
+ throw new ParseException(String.format("Invalid/Unknown RDF format: %s, use one of %s", format, RDFFormat.values()));
+ }
+ } else {
+ fmt = null;
+ }
+
+ // get the files to import.
+ final ArrayList<String> inputFiles = new ArrayList<>();
+ // - provided with the -i option
if (cmd.hasOption('i')) {
inputFiles.addAll(Arrays.asList(cmd.getOptionValues('i')));
}
+ // - or just the filename.
inputFiles.addAll(Arrays.asList(cmd.getArgs()));
-
+
+ // we need at least one file to import
+ // this could be changed...
if (inputFiles.isEmpty()) {
throw new ParseException("No files to import");
}
-
+
+ // initialize the KiWiLoader
log.info("Initializing KiWiLoader for {}; user: {}, password: {}", dbCon, dbUser, String.format("%"+dbPasswd.length()+"s", "*"));
KiWiLoader loader = new KiWiLoader(kiwi, baseUri, context);
loader.setVersioningEnabled(cmd.hasOption("versioning"));
loader.setReasoningEnabled(cmd.hasOption("reasoning"));
loader.initialize();
-
+
log.info("Starting import");
for (String inFile: inputFiles) {
- log.info("Importing {}", inFile);
- long start = System.currentTimeMillis();
- loader.load(inFile, format, gzip);
- long dur = System.currentTimeMillis() - start;
- log.info("Import completed ({}): {}", inFile, dur);
+ try {
+ log.info("Importing {}", inFile);
+ final File f = new File(inFile);
+ String fName = f.getName();
+ InputStream inStream = new FileInputStream(f);
+ if (gzip || inFile.endsWith(".gz")) {
+ log.debug("{} seems to be gzipped", inFile);
+ inStream = new GZIPInputStream(inStream);
+ fName = fName.replaceFirst("\\.gz$", "");
+ }
+
+ long start = System.currentTimeMillis();
+ loader.load(inStream, RDFFormat.forFileName(fName, fmt));
+ long dur = System.currentTimeMillis() - start;
+ log.info("Import completed ({}): {}ms", inFile, dur);
+ } catch (FileNotFoundException e) {
+ log.error("Could not read file {}, skipping...", inFile);
+ } catch (IOException e) {
+ log.error("Error while reading {}: {}", inFile, e);
+ } catch (RDFParseException e) {
+ log.error("file {} contains errors: {}\n{}", inFile, e.getMessage(), e);
+ }
}
-
+
log.info("Import complete, shutting down.");
loader.shutdown();
log.info("Exiting");
} catch (ParseException e) {
log.error(e.getMessage());
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(KiWiLoader.class.getSimpleName(), options, true);
+ printHelp(options);
System.exit(1);
} catch (ConfigurationException e) {
log.error("Could not read system-config.properties: {}", e.getMessage());
@@ -180,14 +285,62 @@ public class KiWiLoader {
}
}
+ private static void printHelp(Options options) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(KiWiLoader.class.getSimpleName(), options, true);
+ }
+
+ /**
+ * Load the triples from the input stream using the provided format.
+ * @param inStream the {@link InputStream} to read from
+ * @param forFileName the expected {@link RDFFormat}
+ * @throws RDFParseException if the stream is invalid (e.g. does not fit with the syntax)
+ * @throws IOException
+ */
+ public void load(InputStream inStream, RDFFormat forFileName) throws RDFParseException, IOException {
+ try {
+ final SailRepositoryConnection con = repository.getConnection();
+ try {
+ con.begin();
+
+ final Resource[] ctx;
+ if (context != null) {
+ ctx = new Resource[] { con.getValueFactory().createURI(context) };
+ } else {
+ ctx = new Resource[] {};
+ }
+
+ con.add(inStream, baseUri, forFileName, ctx);
+
+ con.commit();
+ } finally {
+ con.close();
+ }
+ } catch (RepositoryException re) {
+ log.error("RepositoryException: {}", re.getMessage());
+ }
+ }
+
+ /**
+ * En-/Disable reasoning during load (currently not implemented)
+ * @param enabled
+ */
public void setReasoningEnabled(boolean enabled) {
this.isReasoningEnabled = enabled;
}
+ /**
+ * En-/Disable versioning during load (currently not implemented)
+ * @param enabled
+ */
public void setVersioningEnabled(boolean enabled) {
this.isVersioningEnabled = enabled;
}
+ /**
+ * Shutdown the Repository, close all database connections.
+ * @throws RepositoryException
+ */
public synchronized void shutdown() throws RepositoryException {
if (repository == null || !repository.isInitialized()) {
throw new IllegalStateException("repository was not initialized!");
@@ -196,44 +349,58 @@ public class KiWiLoader {
repository = null;
}
- public void load(String file, String format, boolean gzip) {
- try {
- final SailRepositoryConnection con = repository.getConnection();
- try {
- con.begin();
- // TODO Auto-generated method stub
- con.commit();
- } finally {
- con.close();
- }
- } catch (RepositoryException re) {
- log.error("RepositoryException: {}", re.getMessage());
+ /**
+ * Load the triples from the file using the provided format.
+ * @param file the file to read from
+ * @param format the expected {@link RDFFormat}
+ * @param gzip force using a {@link GZIPInputStream} (will try to guess based on the filename if false)
+ * @throws RDFParseException if the stream is invalid (e.g. does not fit with the syntax)
+ * @throws IOException
+ * @see {@link #load(InputStream, RDFFormat)}
+ */
+ public void load(String file, RDFFormat format, boolean gzip) throws IOException, RDFParseException {
+ final File f = new File(file);
+ InputStream is = new FileInputStream(f);
+ if (gzip || f.getName().endsWith(".gz")) {
+ is = new GZIPInputStream(is);
}
+ load(is, format);
}
+ /**
+ * Initialize the KiWiStore, connect to the database.
+ * @throws RepositoryException
+ */
public synchronized void initialize() throws RepositoryException {
if (repository != null && repository.isInitialized()) {
throw new IllegalStateException("repository already initialized");
}
log.debug("initializing kiwi-store: {}", kiwi);
KiWiStore store = new KiWiStore(kiwi);
-
- TransactionalSail tSail = new KiWiTransactionalSail(store);
- if (isVersioningEnabled) {
- log.debug("enabling versioning...");
- // TODO: Add Versioning
- // tSail = new KiWiVersioningSail(tSail);
- log.warn("versioning not yet supported/implemented");
- }
- if (isReasoningEnabled) {
- log.debug("enabling reasoner...");
- // TODO: Add Reasoning
- // tSail = new KiWiReasoningSail(tSail, null);
- log.warn("reasoning not yet supported/implemented");
+ final Sail sail;
+ if (isVersioningEnabled || isReasoningEnabled) {
+ TransactionalSail tSail = new KiWiTransactionalSail(store);
+ if (isVersioningEnabled) {
+ log.debug("enabling versioning...");
+ // TODO: Add Versioning
+ // tSail = new KiWiVersioningSail(tSail);
+ log.warn("versioning not yet supported/implemented");
+ }
+
+ if (isReasoningEnabled) {
+ log.debug("enabling reasoner...");
+ // TODO: Add Reasoning
+ // tSail = new KiWiReasoningSail(tSail, null);
+ log.warn("reasoning not yet supported/implemented");
+ }
+ sail = tSail;
+ } else {
+ // no transactional sail required here
+ sail = store;
}
-
- repository = new SailRepository(tSail);
+
+ repository = new SailRepository(sail);
repository.initialize();
}
@@ -255,12 +422,12 @@ public class KiWiLoader {
}
}
- private static KiWiDialect getDialect(CommandLine cmd, String dbCon, KiWiDialect dialect)
+ private static KiWiDialect getDialect(CommandLine cmd, String dbCon, KiWiDialect fallback)
throws ParseException {
// get the dialect
final String dbDialect = cmd.getOptionValue('D');
if (!cmd.hasOption('D')) {
- return dialect;
+ return fallback;
} else if (dbDialect == null) {
log.info("No dialect provded, trying to guess based on dbUrl: {}", dbCon);
// try guessing
@@ -287,6 +454,8 @@ public class KiWiLoader {
private static Options buildOptions() {
final Options options = new Options();
+ options.addOption("h", "help", false, "print this help");
+
final Option c = new Option("c", "config", true, "Marmotta system-config.properties file");
c.setArgName("config");
c.setArgs(1);
@@ -303,14 +472,14 @@ public class KiWiLoader {
context.setArgs(1);
context.setOptionalArg(false);
options.addOption(context);
-
+
options.addOption("z", false, "Input file is gzip compressed");
-
+
final Option format = new Option("f", "format", true, "format of rdf file (if guessing based on the extension does not work)");
format.setArgName("mime-type");
format.setArgs(1);
options.addOption(format);
-
+
final Option input = new Option("i", "file", true, "input file(s) to load");
input.setArgName("rdf-file");
input.setArgs(Option.UNLIMITED_VALUES);
@@ -320,27 +489,27 @@ public class KiWiLoader {
dbUrl.setArgName("jdbc-url");
dbUrl.setArgs(1);
options.addOption(dbUrl);
-
+
final Option dbUser = new Option("U", "user", true, "database user");
dbUser.setArgName("dbUser");
options.addOption(dbUser);
-
+
final Option dbPasswd = new Option("P", "dbPasswd", true, "database password");
dbPasswd.setArgName("passwd");
dbPasswd.setArgs(1);
dbPasswd.setOptionalArg(true);
options.addOption(dbPasswd);
-
+
final Option dbDialect = new Option("D", "dbDialect", true, "database dialect (h2, mysql, postgres)");
dbDialect.setArgName("dialect");
options.addOption(dbDialect);
-
+
options.addOption(null, "reasoning", false, "enable reasoning");
options.addOption(null, "versioning", false, "enable versioning");
-
+
return options;
}
-
+
private static String endWith(String string, String suffix) {
return string.replaceFirst("("+Pattern.quote(suffix)+")?$", suffix);
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/188d7533/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/KiWiLoaderTest.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/KiWiLoaderTest.java b/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/KiWiLoaderTest.java
new file mode 100644
index 0000000..b54e75f
--- /dev/null
+++ b/libraries/kiwi/kiwi-loader/src/test/java/org/apache/marmotta/kiwi/loader/KiWiLoaderTest.java
@@ -0,0 +1,64 @@
+package org.apache.marmotta.kiwi.loader;
+
+import static org.junit.Assert.*;
+
+import org.apache.marmotta.kiwi.config.KiWiConfiguration;
+import org.apache.marmotta.kiwi.test.junit.KiWiDatabaseRunner;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.openrdf.repository.RepositoryException;
+
+@RunWith(KiWiDatabaseRunner.class)
+public class KiWiLoaderTest {
+
+ private final KiWiConfiguration kiwiConfig;
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ public KiWiLoaderTest(KiWiConfiguration kiwiConfig) {
+ this.kiwiConfig = kiwiConfig;
+ }
+
+ @Before
+ public void setUp() {
+ // TODO: copy test file to temp folder
+ }
+
+
+ @Test
+ @Ignore("Not yet implemented")
+ public void testMain() {
+ }
+
+ @Test
+ @Ignore("Not yet implemented")
+ public void testLoadFile() throws RepositoryException {
+ KiWiLoader loader = new KiWiLoader(kiwiConfig, "htto://example.com/test/", null);
+ loader.initialize();
+
+// FIXME: loader.load(file, format, gzip);
+
+ loader.shutdown();
+ fail("Not yet implemented");
+ }
+
+ @Test
+ @Ignore("Not yet implemented")
+ public void testLoadInputStream() throws RepositoryException {
+ KiWiLoader loader = new KiWiLoader(kiwiConfig, "htto://example.com/test/", null);
+ loader.initialize();
+
+// FIXME: loader.load(is, format);
+
+ loader.shutdown();
+ fail("Not yet implemented");
+ }
+
+
+
+}