You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2014/02/04 16:07:01 UTC
[1/2] git commit: - loader: support for importing geonames data
Updated Branches:
refs/heads/develop 003f5268b -> 6bcc9078c
- loader: support for importing geonames data
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/47a2b6f1
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/47a2b6f1
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/47a2b6f1
Branch: refs/heads/develop
Commit: 47a2b6f1c02eec3c7383bf7d0b9873dd76c5e992
Parents: c5ff79f
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Tue Feb 4 16:06:39 2014 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Tue Feb 4 16:06:39 2014 +0100
----------------------------------------------------------------------
.../kiwi/loader/generic/KiWiBatchHandler.java | 2 +-
.../kiwi/loader/generic/KiWiHandler.java | 4 +-
loader/marmotta-loader-core/pom.xml | 4 +
.../marmotta/loader/core/MarmottaLoader.java | 46 ++--
.../marmotta/loader/rio/GeonamesFormat.java | 45 ++++
.../marmotta/loader/rio/GeonamesParser.java | 238 +++++++++++++++++++
.../loader/rio/GeonamesParserFactory.java | 46 ++++
.../services/org.openrdf.rio.RDFParserFactory | 1 +
.../loader/core/test/GeonamesParserTest.java | 66 +++++
.../src/test/resources/geonames-sample.txt | 4 +
10 files changed, 436 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiBatchHandler.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiBatchHandler.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiBatchHandler.java
index 23b85f2..cdab0d3 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiBatchHandler.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiBatchHandler.java
@@ -114,7 +114,7 @@ public abstract class KiWiBatchHandler extends KiWiHandler implements RDFHandler
*/
@Override
public void startRDF() throws RDFHandlerException {
- log.info("starting import using optimized {} data loader", backend);
+ log.debug("starting import using optimized {} data loader", backend);
this.tripleBacklog = new ArrayList<>(config.getStatementBatchSize());
this.nodeBacklog = new ArrayList<>(config.getStatementBatchSize()*2);
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
index 4894007..37e5d2c 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
@@ -141,7 +141,7 @@ public class KiWiHandler implements RDFHandler {
throw new RDFHandlerException(e);
}
- log.info("KiWiLoader: RDF bulk import of {} triples finished after {} ms", triples, System.currentTimeMillis() - start);
+ log.debug("KiWiLoader: RDF bulk import of {} triples finished after {} ms", triples, System.currentTimeMillis() - start);
}
/**
@@ -156,7 +156,7 @@ public class KiWiHandler implements RDFHandler {
if(!initialised) {
initialise();
}
- log.info("KiWiLoader: starting RDF bulk import");
+ log.debug("KiWiLoader: starting RDF bulk import");
this.start = System.currentTimeMillis();
this.previous = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/loader/marmotta-loader-core/pom.xml
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/pom.xml b/loader/marmotta-loader-core/pom.xml
index af87efb..146b2fb 100644
--- a/loader/marmotta-loader-core/pom.xml
+++ b/loader/marmotta-loader-core/pom.xml
@@ -59,6 +59,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<!-- Performance Statistics -->
<dependency>
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/core/MarmottaLoader.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/core/MarmottaLoader.java b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/core/MarmottaLoader.java
index 582f508..38e8316 100644
--- a/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/core/MarmottaLoader.java
+++ b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/core/MarmottaLoader.java
@@ -25,6 +25,7 @@ import org.apache.marmotta.loader.api.LoaderHandler;
import org.apache.marmotta.loader.api.LoaderOptions;
import org.apache.marmotta.loader.context.ContextHandler;
import org.apache.marmotta.loader.functions.BackendIdentifierFunction;
+import org.apache.marmotta.loader.rio.GeonamesFormat;
import org.apache.marmotta.loader.statistics.StatisticsHandler;
import org.apache.marmotta.loader.util.DirectoryFilter;
import org.openrdf.model.impl.URIImpl;
@@ -48,6 +49,10 @@ public class MarmottaLoader {
private static Logger log = LoggerFactory.getLogger(MarmottaLoader.class);
+ static {
+ RDFFormat.register(GeonamesFormat.FORMAT);
+ }
+
private Configuration configuration;
@@ -288,23 +293,29 @@ public class MarmottaLoader {
ArchiveEntry entry;
while( (entry = zipStream.getNextEntry()) != null) {
- // detect the file format
- RDFFormat detectedFormat = RDFFormat.forFileName(entry.getName());
- if(format == null) {
- if(detectedFormat != null) {
- log.info("auto-detected entry format: {}", detectedFormat.getName());
- format = detectedFormat;
+ if(! entry.isDirectory()) {
+ log.info("loading entry {} ...", entry.getName());
+
+ // detect the file format
+ RDFFormat detectedFormat = RDFFormat.forFileName(entry.getName());
+ if(format == null) {
+ if(detectedFormat != null) {
+ log.info("auto-detected entry format: {}", detectedFormat.getName());
+ format = detectedFormat;
+ } else {
+ throw new RDFParseException("could not detect input format of entry "+ entry.getName());
+ }
} else {
- throw new RDFParseException("could not detect input format of entry "+ entry.getName());
- }
- } else {
- if(detectedFormat != null && !format.equals(detectedFormat)) {
- log.warn("user-specified format ({}) overrides auto-detected format ({})", format.getName(), detectedFormat.getName());
+ if(detectedFormat != null && !format.equals(detectedFormat)) {
+ log.warn("user-specified entry format ({}) overrides auto-detected format ({})", format.getName(), detectedFormat.getName());
+ } else {
+ log.info("user-specified entry format: {}", format.getName());
+ }
}
- }
- load(zipStream,handler,format);
+ load(zipStream,handler,format);
+ }
}
} else {
@@ -404,6 +415,8 @@ public class MarmottaLoader {
return RDFFormat.RDFXML;
} else if(StringUtils.equalsIgnoreCase(spec,"xml")) {
return RDFFormat.RDFXML;
+ } else if(StringUtils.equalsIgnoreCase(spec,"geonames")) {
+ return GeonamesFormat.FORMAT;
} else if(spec != null) {
return RDFFormat.forMIMEType(spec);
} else {
@@ -449,8 +462,8 @@ public class MarmottaLoader {
OptionBuilder.withArgName("backend")
.hasArgs(1)
.withDescription("backend to use (" + StringUtils.join(Iterators.transform(backends.iterator(), new BackendIdentifierFunction()), ", ") + ")")
- .withLongOpt("backend")
- .create('B');
+ .withLongOpt("backend")
+ .create('B');
options.addOption(backend);
final Option base =
@@ -496,7 +509,6 @@ public class MarmottaLoader {
.withLongOpt("dir")
.create('d');
input.addOption(directories);
- options.addOptionGroup(input);
final Option archives =
OptionBuilder.withArgName("archive")
@@ -574,7 +586,7 @@ public class MarmottaLoader {
throw new ParseException("unrecognized MIME type: " + cmd.getOptionValue('t'));
}
- result.setProperty(LoaderOptions.FORMAT, fmt);
+ result.setProperty(LoaderOptions.FORMAT, fmt.getDefaultMIMEType());
}
if(cmd.hasOption('f')) {
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesFormat.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesFormat.java b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesFormat.java
new file mode 100644
index 0000000..f8059e8
--- /dev/null
+++ b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesFormat.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.loader.rio;
+
+import org.openrdf.rio.RDFFormat;
+
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+/**
+ * Add file description here!
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class GeonamesFormat {
+
+ public static final RDFFormat FORMAT = new RDFFormat(
+ "Geonames",
+ Arrays.asList("text/vnd.geonames.rdf"),
+ Charset.forName("UTF-8"),
+ Arrays.asList("txt"),
+ false,
+ false
+ );
+ static {
+ RDFFormat.register(FORMAT);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesParser.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesParser.java b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesParser.java
new file mode 100644
index 0000000..265222b
--- /dev/null
+++ b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesParser.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.loader.rio;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.LineIterator;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.rio.*;
+import org.openrdf.rio.helpers.RDFParserBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
+
+/**
+ * A specialised RDF parser for Geonames data. Geonames dumps are usually a big text file with a complete
+ * RDF/XML document per line, so the parser will split the lines and then send each line individually to a
+ * wrapped
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class GeonamesParser extends RDFParserBase implements ParseErrorListener {
+
+ private static Logger log = LoggerFactory.getLogger(GeonamesParser.class);
+
+ private RDFParser lineParser;
+
+ private int lineNumber = 0;
+
+ /**
+ * Creates a new RDFParserBase that will use a {@link org.openrdf.model.impl.ValueFactoryImpl} to
+ * create RDF model objects.
+ */
+ public GeonamesParser() {
+ super();
+
+ lineParser = Rio.createParser(RDFFormat.RDFXML);
+ lineParser.setParseErrorListener(this);
+ lineParser.setParserConfig(getParserConfig());
+ }
+
+ /**
+ * Creates a new RDFParserBase that will use the supplied ValueFactory to
+ * create RDF model objects.
+ *
+ * @param valueFactory A ValueFactory.
+ */
+ public GeonamesParser(ValueFactory valueFactory) {
+ super(valueFactory);
+
+ lineParser = Rio.createParser(RDFFormat.RDFXML);
+ lineParser.setParseErrorListener(this);
+ lineParser.setValueFactory(valueFactory);
+ lineParser.setParserConfig(getParserConfig());
+ }
+
+
+
+ /**
+ * Gets the RDF format that this parser can parse.
+ */
+ @Override
+ public RDFFormat getRDFFormat() {
+ return GeonamesFormat.FORMAT;
+ }
+
+ @Override
+ public void setRDFHandler(RDFHandler handler) {
+ super.setRDFHandler(handler);
+
+ lineParser.setRDFHandler(handler);
+ }
+
+
+ @Override
+ public void setParserConfig(ParserConfig config) {
+ super.setParserConfig(config);
+
+ if(lineParser != null) {
+ // called by super.init when lineParser is still null
+ lineParser.setParserConfig(config);
+ }
+ }
+
+ @Override
+ public void setValueFactory(ValueFactory valueFactory) {
+ super.setValueFactory(valueFactory);
+
+ if(lineParser != null) {
+ // called by super.init when lineParser is still null
+ lineParser.setValueFactory(valueFactory);
+ }
+ }
+
+
+ /**
+ * Parses the data from the supplied InputStream, using the supplied baseURI
+ * to resolve any relative URI references.
+ *
+ * @param in The InputStream from which to read the data.
+ * @param baseURI The URI associated with the data in the InputStream.
+ * @throws java.io.IOException If an I/O error occurred while data was read from the InputStream.
+ * @throws org.openrdf.rio.RDFParseException If the parser has found an unrecoverable parse error.
+ * @throws org.openrdf.rio.RDFHandlerException If the configured statement handler has encountered an
+ * unrecoverable error.
+ */
+ @Override
+ public void parse(InputStream in, String baseURI) throws IOException, RDFParseException, RDFHandlerException {
+ LineIterator it = IOUtils.lineIterator(in, RDFFormat.RDFXML.getCharset());
+ try {
+ while(it.hasNext()) {
+ lineNumber++;
+
+ String line = it.nextLine();
+ if(lineNumber % 2 == 0) {
+ // only odd line numbers contain triples
+ StringReader buffer = new StringReader(line);
+ lineParser.parse(buffer, baseURI);
+ }
+ }
+ } finally {
+ it.close();
+ }
+ }
+
+ /**
+ * Parses the data from the supplied Reader, using the supplied baseURI to
+ * resolve any relative URI references.
+ *
+ * @param reader The Reader from which to read the data.
+ * @param baseURI The URI associated with the data in the InputStream.
+ * @throws java.io.IOException If an I/O error occurred while data was read from the InputStream.
+ * @throws org.openrdf.rio.RDFParseException If the parser has found an unrecoverable parse error.
+ * @throws org.openrdf.rio.RDFHandlerException If the configured statement handler has encountered an
+ * unrecoverable error.
+ */
+ @Override
+ public void parse(Reader reader, String baseURI) throws IOException, RDFParseException, RDFHandlerException {
+ LineIterator it = IOUtils.lineIterator(reader);
+ try {
+ while(it.hasNext()) {
+ lineNumber++;
+
+ String line = it.nextLine();
+ if(lineNumber % 2 == 1) {
+ // only odd line numbers contain triples
+ StringReader buffer = new StringReader(line);
+ lineParser.parse(buffer, baseURI);
+ }
+ }
+ } finally {
+ it.close();
+ }
+ }
+
+
+ /**
+ * Reports a warning from the parser. Warning messages are generated
+ * by the parser when it encounters data that is syntactically correct
+ * but which is likely to be a typo. Examples are the use of unknown
+ * or deprecated RDF URIs, e.g. <tt>rdfs:Property</tt> instead of
+ * <tt>rdf:Property</tt>.
+ *
+ * @param msg A warning message.
+ * @param lineNo A line number related to the warning, or -1 if not
+ * available or applicable.
+ * @param colNo A column number related to the warning, or -1 if not
+ */
+ @Override
+ public void warning(String msg, int lineNo, int colNo) {
+ if(getParseErrorListener() != null) {
+ getParseErrorListener().warning(msg, lineNumber, colNo);
+ } else {
+ log.warn("{} [line {}, column {}]", msg, lineNumber, colNo);
+ }
+ }
+
+ /**
+ * Reports an error from the parser. Error messages are generated by
+ * the parser when it encounters an error in the RDF document. The
+ * parser will try its best to recover from the error and continue
+ * parsing when <tt>stopAtFirstError</tt> has been set to
+ * <tt>false</tt>.
+ *
+ * @param msg A error message.
+ * @param lineNo A line number related to the error, or -1 if not
+ * available or applicable.
+ * @param colNo A column number related to the error, or -1 if not
+ * available or applicable.
+ * @see org.openrdf.rio.RDFParser#setStopAtFirstError
+ */
+ @Override
+ public void error(String msg, int lineNo, int colNo) {
+ if(getParseErrorListener() != null) {
+ getParseErrorListener().error(msg, lineNumber, colNo);
+ } else {
+ log.error("{} [line {}, column {}]", msg, lineNumber, colNo);
+ }
+ }
+
+ /**
+ * Reports a fatal error from the parser. A fatal error is an error
+ * of which the RDF parser cannot recover. The parser will stop parsing
+ * directly after it reported the fatal error. Example fatal errors are
+ * unbalanced start- and end-tags in an XML-encoded RDF document.
+ *
+ * @param msg A error message.
+ * @param lineNo A line number related to the error, or -1 if not
+ * available or applicable.
+ * @param colNo A column number related to the error, or -1 if not
+ */
+ @Override
+ public void fatalError(String msg, int lineNo, int colNo) {
+ if(getParseErrorListener() != null) {
+ getParseErrorListener().fatalError(msg, lineNumber, colNo);
+ } else {
+ log.error("{} [line {}, column {}]", msg, lineNumber, colNo);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesParserFactory.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesParserFactory.java b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesParserFactory.java
new file mode 100644
index 0000000..29aa572
--- /dev/null
+++ b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/rio/GeonamesParserFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.loader.rio;
+
+import org.openrdf.rio.RDFFormat;
+import org.openrdf.rio.RDFParser;
+import org.openrdf.rio.RDFParserFactory;
+
+/**
+ * Add file description here!
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class GeonamesParserFactory implements RDFParserFactory {
+
+ /**
+ * Returns the RDF format for this factory.
+ */
+ @Override
+ public RDFFormat getRDFFormat() {
+ return GeonamesFormat.FORMAT;
+ }
+
+ /**
+ * Returns a RDFParser instance.
+ */
+ @Override
+ public RDFParser getParser() {
+ return new GeonamesParser();
+ }
+}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/loader/marmotta-loader-core/src/main/resources/META-INF/services/org.openrdf.rio.RDFParserFactory
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/main/resources/META-INF/services/org.openrdf.rio.RDFParserFactory b/loader/marmotta-loader-core/src/main/resources/META-INF/services/org.openrdf.rio.RDFParserFactory
new file mode 100644
index 0000000..911796b
--- /dev/null
+++ b/loader/marmotta-loader-core/src/main/resources/META-INF/services/org.openrdf.rio.RDFParserFactory
@@ -0,0 +1 @@
+org.apache.marmotta.loader.rio.GeonamesParserFactory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/GeonamesParserTest.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/GeonamesParserTest.java b/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/GeonamesParserTest.java
new file mode 100644
index 0000000..57af9ea
--- /dev/null
+++ b/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/GeonamesParserTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.loader.core.test;
+
+import org.apache.marmotta.loader.rio.GeonamesFormat;
+import org.junit.Assert;
+import org.junit.Test;
+import org.openrdf.model.Statement;
+import org.openrdf.rio.*;
+import org.openrdf.rio.helpers.RDFHandlerBase;
+
+import java.io.IOException;
+
+/**
+ * Add file description here!
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class GeonamesParserTest {
+
+
+ @Test
+ public void testParser() throws RDFParseException, IOException, RDFHandlerException {
+ MockHandler h = new MockHandler();
+
+ RDFParser p = Rio.createParser(GeonamesFormat.FORMAT);
+ p.setRDFHandler(h);
+
+ p.parse(this.getClass().getResourceAsStream("/geonames-sample.txt"), "");
+
+ Assert.assertTrue(h.hasStatements);
+ }
+
+ @Test
+ public void testFormat() throws ClassNotFoundException {
+ Class.forName("org.apache.marmotta.loader.rio.GeonamesFormat");
+
+ RDFFormat f = RDFFormat.forMIMEType("text/vnd.geonames.rdf");
+
+ Assert.assertEquals(GeonamesFormat.FORMAT, f);
+ }
+
+ private static class MockHandler extends RDFHandlerBase {
+ boolean hasStatements = false;
+
+ @Override
+ public void handleStatement(Statement st) throws RDFHandlerException {
+ hasStatements = true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/47a2b6f1/loader/marmotta-loader-core/src/test/resources/geonames-sample.txt
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/test/resources/geonames-sample.txt b/loader/marmotta-loader-core/src/test/resources/geonames-sample.txt
new file mode 100644
index 0000000..24edb66
--- /dev/null
+++ b/loader/marmotta-loader-core/src/test/resources/geonames-sample.txt
@@ -0,0 +1,4 @@
+http://sws.geonames.org/1/
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><rdf:RDF xmlns:cc="http://creativecommons.org/ns#" xmlns:dcterms="http://purl.org/dc/terms/" xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:gn="http://www.geonames.org/ontology#" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:wgs84_pos="http://www.w3.org/2003/01/geo/wgs84_pos#"><gn:Feature rdf:about="http://sws.geonames.org/1/"><rdfs:isDefinedBy>http://sws.geonames.org/1/about.rdf</rdfs:isDefinedBy><gn:name>Kūh-e Zardar</gn:name><gn:alternateName xml:lang="fa">Kūh-e Zardar</gn:alternateName><gn:featureClass rdf:resource="http://www.geonames.org/ontology#T"/><gn:featureCode rdf:resource="http://www.geonames.org/ontology#T.MT"/><gn:countryCode>IR</gn:countryCode><wgs84_pos:lat>32.98333</wgs84_pos:lat><wgs84_pos:long>49.13333</wgs84_pos:long><gn:parentFeature rdf:resource="http://sws.geonames.org/127082/"/><gn:parentCountry
rdf:resource="http://sws.geonames.org/130758/"/><gn:parentADM1 rdf:resource="http://sws.geonames.org/127082/"/><gn:nearbyFeatures rdf:resource="http://sws.geonames.org/1/nearby.rdf"/><gn:locationMap rdf:resource="http://www.geonames.org/1/kuh-e-zardar.html"/></gn:Feature></rdf:RDF>
+http://sws.geonames.org/3/
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><rdf:RDF xmlns:cc="http://creativecommons.org/ns#" xmlns:dcterms="http://purl.org/dc/terms/" xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:gn="http://www.geonames.org/ontology#" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:wgs84_pos="http://www.w3.org/2003/01/geo/wgs84_pos#"><gn:Feature rdf:about="http://sws.geonames.org/3/"><rdfs:isDefinedBy>http://sws.geonames.org/3/about.rdf</rdfs:isDefinedBy><gn:name>Zamīn Sūkhteh</gn:name><gn:alternateName xml:lang="fa">Zamīn Sūkhteh</gn:alternateName><gn:featureClass rdf:resource="http://www.geonames.org/ontology#P"/><gn:featureCode rdf:resource="http://www.geonames.org/ontology#P.PPL"/><gn:countryCode>IR</gn:countryCode><wgs84_pos:lat>32.48333</wgs84_pos:lat><wgs84_pos:long>48.91667</wgs84_pos:long><gn:parentFeature rdf:resource="http://sws.geonames.org/127082/"/><gn:parentCou
ntry rdf:resource="http://sws.geonames.org/130758/"/><gn:parentADM1 rdf:resource="http://sws.geonames.org/127082/"/><gn:nearbyFeatures rdf:resource="http://sws.geonames.org/3/nearby.rdf"/><gn:locationMap rdf:resource="http://www.geonames.org/3/zamin-sukhteh.html"/></gn:Feature></rdf:RDF>
\ No newline at end of file
[2/2] git commit: Merge remote-tracking branch 'origin/develop' into
develop
Posted by ss...@apache.org.
Merge remote-tracking branch 'origin/develop' into develop
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/6bcc9078
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/6bcc9078
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/6bcc9078
Branch: refs/heads/develop
Commit: 6bcc9078c9a18d616cd554882c82c89001348376
Parents: 47a2b6f 003f526
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Tue Feb 4 16:06:51 2014 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Tue Feb 4 16:06:51 2014 +0100
----------------------------------------------------------------------
.../marmotta-rio-api/pom.xml | 12 +
.../marmotta-rio-ical/pom.xml | 12 +
.../marmotta-rio-rss/pom.xml | 7 +
.../marmotta-rio-vcard/pom.xml | 12 +
.../marmotta-sail-contextaware/pom.xml | 12 +
.../marmotta-sail-transactions/pom.xml | 12 +
.../marmotta-util-facading/pom.xml | 12 +
.../marmotta-util-filter/pom.xml | 12 +
.../marmotta-util-tripletable/pom.xml | 12 +
extras/webjars/sgvizler/pom.xml | 8 +
.../webjars/sgvizler/0.5.1/lib/d3.v2.min.js | 1 -
.../0.5.1/lib/raphael-dracula.pack.min.js | 1 -
.../webjars/sgvizler/0.5.1/sgvizler.chart.css | 1 -
.../webjars/sgvizler/0.5.1/sgvizler.js | 1 -
extras/webjars/snorql/pom.xml | 9 +-
.../snorql/src/main/resources/namespaces.js | 24 +-
.../snorql/src/main/resources/prototype.js | 17 -
.../src/main/resources/scriptaculous/builder.js | 17 -
.../main/resources/scriptaculous/controls.js | 30 +-
.../main/resources/scriptaculous/dragdrop.js | 23 +-
.../src/main/resources/scriptaculous/effects.js | 28 +-
.../resources/scriptaculous/scriptaculous.js | 52 ++-
.../src/main/resources/scriptaculous/slider.js | 22 +-
.../src/main/resources/scriptaculous/sound.js | 24 +-
.../main/resources/scriptaculous/unittest.js | 24 +-
.../webjars/snorql/src/main/resources/snorql.js | 17 -
.../webjars/snorql/src/main/resources/sparql.js | 75 ++--
.../snorql/src/main/resources/xml-to-html.xsl | 366 +++++++++----------
extras/webjars/strftime/pom.xml | 7 +
libraries/ldclient/ldclient-api/pom.xml | 12 +
libraries/ldclient/ldclient-core/pom.xml | 7 +
.../ldclient/ldclient-provider-rdf/pom.xml | 12 +
parent/pom.xml | 9 +
platform/marmotta-versioning-common/pom.xml | 3 +-
34 files changed, 467 insertions(+), 426 deletions(-)
----------------------------------------------------------------------