You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2014/01/09 21:22:20 UTC
[2/2] git commit: - changed LDClient so it uses the more lightweight
Sesame Model API introduced in Sesame 2.7 - cleaned up tests for LDClient
- changed LDClient so it uses the more lightweight Sesame Model API introduced in Sesame 2.7
- cleaned up tests for LDClient
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/d10a5c60
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/d10a5c60
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/d10a5c60
Branch: refs/heads/develop
Commit: d10a5c6077d3609de677728cf944e60ff802c854
Parents: 4aa89b0
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Thu Jan 9 21:22:11 2014 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Thu Jan 9 21:22:11 2014 +0100
----------------------------------------------------------------------
commons/marmotta-commons/pom.xml | 12 +-
.../commons/sesame/model/ModelCommons.java | 204 +++++++++++++++++++
.../services/test/dummy/DummyProvider.java | 57 ++----
.../services/test/dummy/DummyProvider.java | 40 ++--
.../marmotta/ldcache/services/LDCache.java | 20 +-
.../ldcache/sail/test/dummy/DummyProvider.java | 40 ++--
.../ldcache/sail/test/dummy/DummyProvider.java | 40 ++--
.../marmotta/ldclient/model/ClientResponse.java | 42 ++--
libraries/ldclient/ldclient-core/pom.xml | 1 +
.../services/provider/AbstractHttpProvider.java | 29 +--
.../services/provider/BlockingProvider.java | 14 +-
.../test/provider/ProviderTestBase.java | 103 ++++++++++
.../facebook/FacebookGraphProvider.java | 162 +++++++--------
.../test/facebook/FacebookProviderTest.java | 129 +-----------
.../provider/html/AbstractHTMLDataProvider.java | 30 +--
.../provider/ldap/LdapFoafProvider.java | 171 +++++++---------
.../provider/mediawiki/MediawikiProvider.java | 122 +++++------
.../test/mediawiki/TestMediawikiProvider.java | 74 +------
.../provider/rdf/AbstractRDFProvider.java | 54 ++---
.../ldclient/provider/rdf/SPARQLProvider.java | 77 +++----
.../test/rdf/TestLinkedDataProvider.java | 101 +--------
.../ldclient/test/rdf/TestSPARQLProvider.java | 7 +-
.../ldclient/test/rdf/TestStanbolProvider.java | 3 +-
.../ldclient/test/vimeo/TestVimeoProvider.java | 104 +---------
.../provider/xml/AbstractXMLDataProvider.java | 57 ++----
.../youtube/YoutubeVideoPagesProvider.java | 34 +---
.../test/youtube/TestYoutubeProvider.java | 161 +--------------
.../LinkedDataCachingWebService.java | 43 ++--
28 files changed, 732 insertions(+), 1199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/commons/marmotta-commons/pom.xml
----------------------------------------------------------------------
diff --git a/commons/marmotta-commons/pom.xml b/commons/marmotta-commons/pom.xml
index eaf49bf..282f39d 100644
--- a/commons/marmotta-commons/pom.xml
+++ b/commons/marmotta-commons/pom.xml
@@ -106,12 +106,13 @@
<groupId>org.openrdf.sesame</groupId>
<artifactId>sesame-queryresultio-api</artifactId>
</dependency>
-
<dependency>
- <!-- TODO: this should be removed -->
+ <groupId>org.openrdf.sesame</groupId>
+ <artifactId>sesame-sail-memory</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.openrdf.sesame</groupId>
<artifactId>sesame-repository-sail</artifactId>
- <optional>true</optional>
</dependency>
@@ -122,11 +123,6 @@
</dependency>
<dependency>
<groupId>org.openrdf.sesame</groupId>
- <artifactId>sesame-sail-memory</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.openrdf.sesame</groupId>
<artifactId>sesame-rio-rdfxml</artifactId>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/ModelCommons.java
----------------------------------------------------------------------
diff --git a/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/ModelCommons.java b/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/ModelCommons.java
new file mode 100644
index 0000000..a51f7a3
--- /dev/null
+++ b/commons/marmotta-commons/src/main/java/org/apache/marmotta/commons/sesame/model/ModelCommons.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.commons.sesame.model;
+
+import javolution.util.function.Predicate;
+import org.openrdf.model.Model;
+import org.openrdf.model.Statement;
+import org.openrdf.model.impl.TreeModel;
+import org.openrdf.repository.Repository;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+import org.openrdf.repository.RepositoryResult;
+import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.rio.*;
+import org.openrdf.rio.helpers.RDFHandlerBase;
+import org.openrdf.sail.memory.MemoryStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+
+/**
+ * Utilities for working with Sesame Models
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class ModelCommons {
+
+ private static Logger log = LoggerFactory.getLogger(ModelCommons.class);
+
+ /**
+ * Create an instance of RDFHandler using the model passed as parameter as underlying triple store.
+ * @param model the model to wrap in an RDFHandler
+ * @param filters an optional list of filters; if any of the filters rejects the statement it is not added
+ * @return
+ */
+ public static RDFHandler createModelHandler(final Model model, final Predicate<Statement>... filters) {
+ return new RDFHandlerBase() {
+ @Override
+ public void handleStatement(Statement st) throws RDFHandlerException {
+ for(Predicate<Statement> f : filters) {
+ if(!f.test(st)) {
+ return;
+ }
+ }
+ model.add(st);
+ }
+ };
+ }
+
+ /**
+ * Add statements from the given input stream to the given model. Similar to RepositoryConnection.add, just works
+ * directly on a model.
+ *
+ * @param model the model to add the statements to
+ * @param in input stream to read the statements from
+ * @param baseURI base URI to resolve relative URIs
+ * @param format RDF format of the data in the input stream
+ * @throws IOException
+ * @throws RDFParseException
+ */
+ public static void add(Model model, InputStream in, String baseURI, RDFFormat format, Predicate<Statement>... filters) throws IOException, RDFParseException {
+ try {
+
+ RDFParser parser = Rio.createParser(format);
+ parser.setRDFHandler(createModelHandler(model, filters));
+ parser.parse(in, baseURI);
+
+ } catch (RDFHandlerException e) {
+ log.error("RepositoryException:", e);
+ }
+
+ }
+
+
+ /**
+ * Add statements from the given reader to the given model. Similar to RepositoryConnection.add, just works
+ * directly on a model.
+ *
+ * @param model the model to add the statements to
+ * @param in reader to read the statements from
+ * @param baseURI base URI to resolve relative URIs
+ * @param format RDF format of the data in the reader
+ * @throws IOException
+ * @throws RDFParseException
+ */
+ public static void add(Model model, Reader in, String baseURI, RDFFormat format, Predicate<Statement>... filters) throws IOException, RDFParseException {
+ try {
+
+ RDFParser parser = Rio.createParser(format);
+ parser.setRDFHandler(createModelHandler(model, filters));
+ parser.parse(in, baseURI);
+
+ } catch (RDFHandlerException e) {
+ log.error("RepositoryException:", e);
+ }
+
+ }
+
+
+ /**
+ * Export all triples in the model passed as argument to the RDF handler passed as second argument. Similar to
+ * RepositoryConnection.export.
+ *
+ * @param model
+ * @param handler
+ * @throws RDFHandlerException
+ */
+ public static void export(Model model, RDFHandler handler) throws RDFHandlerException {
+ handler.startRDF();
+ for(Statement stmt : model) {
+ handler.handleStatement(stmt);
+ }
+ handler.endRDF();
+ }
+
+
+ /**
+ * Copy the contents of the model over to a newly initialised in-memory repository.
+ *
+ * @param model the model to wrap in a memory repository
+ * @return the memory repository
+ */
+ public static Repository asRepository(Model model) throws RepositoryException {
+ Repository repository = new SailRepository(new MemoryStore());
+ repository.initialize();
+ RepositoryConnection con = repository.getConnection();
+ try {
+ con.begin();
+
+ con.add(model);
+
+ con.commit();
+ } catch(RepositoryException ex) {
+ con.rollback();
+ } finally {
+ con.close();
+ }
+
+ return repository;
+
+ }
+
+
+ /**
+ * Copy the contents of the given repository over to a newly created model, optionally applying the filters given
+ * as variable argument.
+ *
+ * @param repository
+ * @return
+ * @throws RepositoryException
+ */
+ public static Model asModel(Repository repository, Predicate<Statement>... filters) throws RepositoryException {
+ Model model = new TreeModel();
+
+ RepositoryConnection con = repository.getConnection();
+ try {
+ con.begin();
+
+ RepositoryResult<Statement> r = con.getStatements(null,null,null,true);
+ try {
+ rloop: while(r.hasNext()) {
+ Statement st = r.next();
+
+ for(Predicate<Statement> f : filters) {
+ if(!f.test(st)) {
+ continue rloop;
+ }
+ }
+
+ model.add(st);
+ }
+ } finally {
+ r.close();
+ }
+
+ con.commit();
+ } catch(RepositoryException ex) {
+ con.rollback();
+ } finally {
+ con.close();
+ }
+
+ return model;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldcache/ldcache-backend-infinispan/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-backend-infinispan/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java b/libraries/ldcache/ldcache-backend-infinispan/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
index 4b6f92a..b789118 100644
--- a/libraries/ldcache/ldcache-backend-infinispan/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
+++ b/libraries/ldcache/ldcache-backend-infinispan/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
@@ -1,13 +1,12 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,18 +16,16 @@
*/
package org.apache.marmotta.ldcache.services.test.dummy;
+import org.apache.marmotta.commons.sesame.model.ModelCommons;
import org.apache.marmotta.ldclient.api.endpoint.Endpoint;
import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
import org.apache.marmotta.ldclient.api.provider.DataProvider;
import org.apache.marmotta.ldclient.exception.DataRetrievalException;
import org.apache.marmotta.ldclient.model.ClientResponse;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.model.Model;
+import org.openrdf.model.impl.TreeModel;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFParseException;
-import org.openrdf.sail.memory.MemoryStore;
import java.io.IOException;
@@ -48,34 +45,20 @@ public class DummyProvider implements DataProvider {
public ClientResponse retrieveResource(String resource, LDClientService client, Endpoint endpoint) throws DataRetrievalException {
String filename = resource.substring("http://localhost/".length()) + ".ttl";
+ Model triples = new TreeModel();
try {
- Repository triples = new SailRepository(new MemoryStore());
- triples.initialize();
+ ModelCommons.add(triples, DummyProvider.class.getResourceAsStream(filename), resource, RDFFormat.TURTLE);
- RepositoryConnection con = triples.getConnection();
- try {
- con.begin();
-
- con.add(DummyProvider.class.getResourceAsStream(filename), resource, RDFFormat.TURTLE);
-
- con.commit();
- } catch(RepositoryException ex) {
- con.rollback();
- } catch (RDFParseException e) {
- throw new DataRetrievalException("could not parse resource data for file "+filename);
- } catch (IOException e) {
- throw new DataRetrievalException("could not load resource data for file "+filename);
- } finally {
- con.close();
- }
-
- ClientResponse response = new ClientResponse(200, triples);
-
- return response;
- } catch (RepositoryException e) {
+ } catch (RDFParseException e) {
+ throw new DataRetrievalException("could not parse resource data for file "+filename);
+ } catch (IOException e) {
throw new DataRetrievalException("could not load resource data for file "+filename);
}
+ ClientResponse response = new ClientResponse(200, triples);
+
+ return response;
+
}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldcache/ldcache-backend-kiwi/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-backend-kiwi/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java b/libraries/ldcache/ldcache-backend-kiwi/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
index 4b6f92a..685c79a 100644
--- a/libraries/ldcache/ldcache-backend-kiwi/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
+++ b/libraries/ldcache/ldcache-backend-kiwi/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
@@ -17,18 +17,16 @@
*/
package org.apache.marmotta.ldcache.services.test.dummy;
+import org.apache.marmotta.commons.sesame.model.ModelCommons;
import org.apache.marmotta.ldclient.api.endpoint.Endpoint;
import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
import org.apache.marmotta.ldclient.api.provider.DataProvider;
import org.apache.marmotta.ldclient.exception.DataRetrievalException;
import org.apache.marmotta.ldclient.model.ClientResponse;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.model.Model;
+import org.openrdf.model.impl.TreeModel;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFParseException;
-import org.openrdf.sail.memory.MemoryStore;
import java.io.IOException;
@@ -48,34 +46,20 @@ public class DummyProvider implements DataProvider {
public ClientResponse retrieveResource(String resource, LDClientService client, Endpoint endpoint) throws DataRetrievalException {
String filename = resource.substring("http://localhost/".length()) + ".ttl";
+ Model triples = new TreeModel();
try {
- Repository triples = new SailRepository(new MemoryStore());
- triples.initialize();
+ ModelCommons.add(triples, DummyProvider.class.getResourceAsStream(filename), resource, RDFFormat.TURTLE);
- RepositoryConnection con = triples.getConnection();
- try {
- con.begin();
-
- con.add(DummyProvider.class.getResourceAsStream(filename), resource, RDFFormat.TURTLE);
-
- con.commit();
- } catch(RepositoryException ex) {
- con.rollback();
- } catch (RDFParseException e) {
- throw new DataRetrievalException("could not parse resource data for file "+filename);
- } catch (IOException e) {
- throw new DataRetrievalException("could not load resource data for file "+filename);
- } finally {
- con.close();
- }
-
- ClientResponse response = new ClientResponse(200, triples);
-
- return response;
- } catch (RepositoryException e) {
+ } catch (RDFParseException e) {
+ throw new DataRetrievalException("could not parse resource data for file "+filename);
+ } catch (IOException e) {
throw new DataRetrievalException("could not load resource data for file "+filename);
}
+ ClientResponse response = new ClientResponse(200, triples);
+
+ return response;
+
}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldcache/ldcache-core/src/main/java/org/apache/marmotta/ldcache/services/LDCache.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-core/src/main/java/org/apache/marmotta/ldcache/services/LDCache.java b/libraries/ldcache/ldcache-core/src/main/java/org/apache/marmotta/ldcache/services/LDCache.java
index a01842f..1b66511 100644
--- a/libraries/ldcache/ldcache-core/src/main/java/org/apache/marmotta/ldcache/services/LDCache.java
+++ b/libraries/ldcache/ldcache-core/src/main/java/org/apache/marmotta/ldcache/services/LDCache.java
@@ -278,23 +278,7 @@ public class LDCache implements LDCachingService {
try {
URI subject = cacheConnection1.getValueFactory().createURI(resource.stringValue());
- RepositoryConnection respConnection = response.getTriples().getConnection();
-
- cacheConnection1.remove(subject, null, null);
-
- int count = 0;
- RepositoryResult<Statement> triples = respConnection.getStatements(null,null,null,true);
- while(triples.hasNext()) {
- Statement triple = triples.next();
- try {
- cacheConnection1.add(triple);
- } catch (RuntimeException ex) {
- log.warn("not adding triple {}: an exception occurred ({})",triple,ex.getMessage());
- }
- count++;
- }
- triples.close();
- respConnection.close();
+ cacheConnection1.add(response.getData());
CacheEntry newEntry = new CacheEntry();
newEntry.setResource(subject);
@@ -305,7 +289,7 @@ public class LDCache implements LDCachingService {
} else {
newEntry.setUpdateCount(1);
}
- newEntry.setTripleCount(count);
+ newEntry.setTripleCount(response.getData().size());
cacheConnection1.removeCacheEntry(resource);
cacheConnection1.addCacheEntry(resource, newEntry);
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java b/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
index 15f3d81..7594dde 100644
--- a/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
+++ b/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
@@ -17,18 +17,16 @@
*/
package org.apache.marmotta.ldcache.sail.test.dummy;
+import org.apache.marmotta.commons.sesame.model.ModelCommons;
import org.apache.marmotta.ldclient.api.endpoint.Endpoint;
import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
import org.apache.marmotta.ldclient.api.provider.DataProvider;
import org.apache.marmotta.ldclient.exception.DataRetrievalException;
import org.apache.marmotta.ldclient.model.ClientResponse;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.model.Model;
+import org.openrdf.model.impl.TreeModel;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFParseException;
-import org.openrdf.sail.memory.MemoryStore;
import java.io.IOException;
@@ -48,34 +46,20 @@ public class DummyProvider implements DataProvider {
public ClientResponse retrieveResource(String resource, LDClientService client, Endpoint endpoint) throws DataRetrievalException {
String filename = resource.substring("http://remote/".length()) + ".ttl";
+ Model triples = new TreeModel();
try {
- Repository triples = new SailRepository(new MemoryStore());
- triples.initialize();
+ ModelCommons.add(triples, DummyProvider.class.getResourceAsStream(filename), resource, RDFFormat.TURTLE);
- RepositoryConnection con = triples.getConnection();
- try {
- con.begin();
-
- con.add(DummyProvider.class.getResourceAsStream(filename), resource, RDFFormat.TURTLE);
-
- con.commit();
- } catch(RepositoryException ex) {
- con.rollback();
- } catch (RDFParseException e) {
- throw new DataRetrievalException("could not parse resource data for file "+filename);
- } catch (IOException e) {
- throw new DataRetrievalException("could not load resource data for file "+filename);
- } finally {
- con.close();
- }
-
- ClientResponse response = new ClientResponse(200, triples);
-
- return response;
- } catch (RepositoryException e) {
+ } catch (RDFParseException e) {
+ throw new DataRetrievalException("could not parse resource data for file "+filename);
+ } catch (IOException e) {
throw new DataRetrievalException("could not load resource data for file "+filename);
}
+ ClientResponse response = new ClientResponse(200, triples);
+
+ return response;
+
}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java b/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
index 15f3d81..7594dde 100644
--- a/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
+++ b/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
@@ -17,18 +17,16 @@
*/
package org.apache.marmotta.ldcache.sail.test.dummy;
+import org.apache.marmotta.commons.sesame.model.ModelCommons;
import org.apache.marmotta.ldclient.api.endpoint.Endpoint;
import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
import org.apache.marmotta.ldclient.api.provider.DataProvider;
import org.apache.marmotta.ldclient.exception.DataRetrievalException;
import org.apache.marmotta.ldclient.model.ClientResponse;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.model.Model;
+import org.openrdf.model.impl.TreeModel;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFParseException;
-import org.openrdf.sail.memory.MemoryStore;
import java.io.IOException;
@@ -48,34 +46,20 @@ public class DummyProvider implements DataProvider {
public ClientResponse retrieveResource(String resource, LDClientService client, Endpoint endpoint) throws DataRetrievalException {
String filename = resource.substring("http://remote/".length()) + ".ttl";
+ Model triples = new TreeModel();
try {
- Repository triples = new SailRepository(new MemoryStore());
- triples.initialize();
+ ModelCommons.add(triples, DummyProvider.class.getResourceAsStream(filename), resource, RDFFormat.TURTLE);
- RepositoryConnection con = triples.getConnection();
- try {
- con.begin();
-
- con.add(DummyProvider.class.getResourceAsStream(filename), resource, RDFFormat.TURTLE);
-
- con.commit();
- } catch(RepositoryException ex) {
- con.rollback();
- } catch (RDFParseException e) {
- throw new DataRetrievalException("could not parse resource data for file "+filename);
- } catch (IOException e) {
- throw new DataRetrievalException("could not load resource data for file "+filename);
- } finally {
- con.close();
- }
-
- ClientResponse response = new ClientResponse(200, triples);
-
- return response;
- } catch (RepositoryException e) {
+ } catch (RDFParseException e) {
+ throw new DataRetrievalException("could not parse resource data for file "+filename);
+ } catch (IOException e) {
throw new DataRetrievalException("could not load resource data for file "+filename);
}
+ ClientResponse response = new ClientResponse(200, triples);
+
+ return response;
+
}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java b/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java
index 70ebf48..68ad7e7 100644
--- a/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java
+++ b/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java
@@ -18,20 +18,24 @@
package org.apache.marmotta.ldclient.model;
import org.apache.commons.lang3.time.DateUtils;
+import org.apache.marmotta.commons.sesame.model.ModelCommons;
+import org.openrdf.model.Model;
+import org.openrdf.model.impl.TreeModel;
import org.openrdf.repository.Repository;
+import org.openrdf.repository.RepositoryException;
import java.util.Date;
/**
* LDCache Client Response
- *
+ *
* @author Sebastian Schaffert
* @author Sergio Fernández
*/
public class ClientResponse {
- private static final int DEFAULT_HTTP_CODE = 200;
-
+ private static final int DEFAULT_HTTP_CODE = 200;
+
private static final int DEFAULT_EXPIRATION_IN_DAYS = 7;
/**
@@ -39,26 +43,40 @@ public class ClientResponse {
*/
private int httpStatus;
- private Repository triples;
+ private Model data;
private Date expires;
-
- public ClientResponse(Repository triples) {
- this(DEFAULT_HTTP_CODE, triples);
- }
+ @Deprecated
public ClientResponse(int httpStatus, Repository triples) {
- this.triples = triples;
this.expires = DateUtils.addDays(new Date(), DEFAULT_EXPIRATION_IN_DAYS);
this.httpStatus = httpStatus;
+
+ try {
+ this.data = ModelCommons.asModel(triples);
+ } catch (RepositoryException e) {
+ this.data = new TreeModel();
+ }
}
+ public ClientResponse(int httpStatus, Model triples) {
+ this.data = triples;
+ this.expires = DateUtils.addDays(new Date(), DEFAULT_EXPIRATION_IN_DAYS);
+ this.httpStatus = httpStatus;
+ }
+
+
+ @Deprecated
public Repository getTriples() {
- return triples;
+ try {
+ return ModelCommons.asRepository(this.data);
+ } catch (RepositoryException e) {
+ return null;
+ }
}
- public void setTriples(Repository triples) {
- this.triples = triples;
+ public Model getData() {
+ return data;
}
public Date getExpires() {
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-core/pom.xml
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-core/pom.xml b/libraries/ldclient/ldclient-core/pom.xml
index 9445be8..31494f6 100644
--- a/libraries/ldclient/ldclient-core/pom.xml
+++ b/libraries/ldclient/ldclient-core/pom.xml
@@ -47,6 +47,7 @@
<configuration>
<includes>
<include>**/test/helper/**</include>
+ <include>**/test/provider/**</include>
</includes>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java b/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java
index 8f45da0..5a41a60 100644
--- a/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java
+++ b/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java
@@ -32,24 +32,15 @@ import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
import org.apache.marmotta.ldclient.api.provider.DataProvider;
import org.apache.marmotta.ldclient.exception.DataRetrievalException;
import org.apache.marmotta.ldclient.model.ClientResponse;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.model.Model;
+import org.openrdf.model.impl.TreeModel;
import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepository;
-import org.openrdf.sail.memory.MemoryStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-import java.util.Set;
+import java.util.*;
import static org.apache.marmotta.commons.http.MarmottaHttpUtils.parseContentType;
@@ -85,14 +76,15 @@ public abstract class AbstractHttpProvider implements DataProvider {
*
*
*
+ *
* @param resourceUri
- * @param repository an RDF repository for storing an RDF representation of the dataset located at the remote resource.
+ * @param model an RDF repository for storing an RDF representation of the dataset located at the remote resource.
* @param in input stream as returned by the remote webservice
* @param contentType content type as returned in the HTTP headers of the remote webservice
* @return a possibly empty list of URLs of additional resources to retrieve to complete the content
* @throws java.io.IOException in case an error occurs while reading the input stream
*/
- protected abstract List<String> parseResponse(String resourceUri, String requestUrl, Repository repository, InputStream in, String contentType) throws DataRetrievalException;
+ protected abstract List<String> parseResponse(String resourceUri, String requestUrl, Model model, InputStream in, String contentType) throws DataRetrievalException;
/**
* Retrieve the data for a resource using the given http client and endpoint definition. The service is
@@ -171,9 +163,7 @@ public abstract class AbstractHttpProvider implements DataProvider {
}
if(log.isInfoEnabled()) {
- RepositoryConnection con = handler.triples.getConnection();
- log.info("retrieved {} triples for resource {}; expiry date: {}",new Object[] {con.size(),resource,expiresDate});
- con.close();
+ log.info("retrieved {} triples for resource {}; expiry date: {}", new Object[]{handler.triples.size(), resource, expiresDate});
}
ClientResponse result = new ClientResponse(handler.httpStatus, handler.triples);
@@ -223,7 +213,7 @@ public abstract class AbstractHttpProvider implements DataProvider {
private String requestUrl;
// the repository where the triples will be stored in case the data providers return them
- private final Repository triples;
+ private final Model triples;
private final Endpoint endpoint;
@@ -235,8 +225,7 @@ public abstract class AbstractHttpProvider implements DataProvider {
this.resource = resource;
this.endpoint = endpoint;
- triples = new SailRepository(new MemoryStore());
- triples.initialize();
+ triples = new TreeModel();
}
@Override
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java b/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java
index ae75d59..fac061d 100644
--- a/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java
+++ b/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java
@@ -22,10 +22,8 @@ import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
import org.apache.marmotta.ldclient.api.provider.DataProvider;
import org.apache.marmotta.ldclient.exception.DataRetrievalException;
import org.apache.marmotta.ldclient.model.ClientResponse;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepository;
-import org.openrdf.sail.memory.MemoryStore;
+import org.openrdf.model.Model;
+import org.openrdf.model.impl.TreeModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,13 +41,7 @@ public class BlockingProvider implements DataProvider {
private static Logger log = LoggerFactory.getLogger(BlockingProvider.class);
- private static final Repository empty_repository = new SailRepository(new MemoryStore());
- static {
- try {
- empty_repository.initialize();
- } catch (RepositoryException e) {
- }
- }
+ private static final Model empty_repository = new TreeModel();
/**
* Return the name of this data provider. To be used e.g. in the configuration and in log messages.
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-core/src/test/java/org/apache/marmotta/ldclient/test/provider/ProviderTestBase.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-core/src/test/java/org/apache/marmotta/ldclient/test/provider/ProviderTestBase.java b/libraries/ldclient/ldclient-core/src/test/java/org/apache/marmotta/ldclient/test/provider/ProviderTestBase.java
new file mode 100644
index 0000000..a290d9c
--- /dev/null
+++ b/libraries/ldclient/ldclient-core/src/test/java/org/apache/marmotta/ldclient/test/provider/ProviderTestBase.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.ldclient.test.provider;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.marmotta.commons.sesame.model.ModelCommons;
+import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
+import org.apache.marmotta.ldclient.model.ClientResponse;
+import org.apache.marmotta.ldclient.services.ldclient.LDClient;
+import org.apache.marmotta.ldclient.test.helper.TestLDClient;
+import org.junit.*;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.openrdf.query.BooleanQuery;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.rio.RDFFormat;
+import org.openrdf.rio.Rio;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.io.StringWriter;
+
+/**
+ * Add file description here!
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class ProviderTestBase {
+
+ protected LDClientService ldclient;
+
+ private static Logger log = LoggerFactory.getLogger(ProviderTestBase.class);
+
+ @Before
+ public void setupClient() {
+ ldclient = new TestLDClient(new LDClient());
+ }
+
+ @After
+ public void shutdownClient() {
+ ldclient.shutdown();
+ }
+
+ final Logger logger =
+ LoggerFactory.getLogger(this.getClass());
+
+ @Rule
+ public TestWatcher watchman = new TestWatcher() {
+ /**
+ * Invoked when a test is about to start
+ */
+ @Override
+ protected void starting(Description description) {
+ logger.info("{} being run...", description.getMethodName());
+ }
+ };
+
+
+
+ protected void testResource(String uri, String sparqlFile) throws Exception {
+
+ Assume.assumeTrue(ldclient.ping(uri));
+
+ ClientResponse response = ldclient.retrieveResource(uri);
+
+ RepositoryConnection connection = ModelCommons.asRepository(response.getData()).getConnection();
+ connection.begin();
+ Assert.assertTrue(connection.size() > 0);
+
+ // run a SPARQL test to see if the returned data is correct
+ InputStream sparql = this.getClass().getResourceAsStream(sparqlFile);
+ BooleanQuery testLabel = connection.prepareBooleanQuery(QueryLanguage.SPARQL, IOUtils.toString(sparql));
+ Assert.assertTrue("SPARQL test query failed", testLabel.evaluate());
+
+ if(log.isDebugEnabled()) {
+ StringWriter out = new StringWriter();
+ connection.export(Rio.createWriter(RDFFormat.TURTLE, out));
+ log.debug("DATA:");
+ log.debug(out.toString());
+ }
+
+ connection.commit();
+ connection.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java b/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java
index 9da22a7..92fd837 100644
--- a/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java
+++ b/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java
@@ -17,17 +17,10 @@
package org.apache.marmotta.ldclient.provider.facebook;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -44,22 +37,22 @@ import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
import org.apache.marmotta.ldclient.api.provider.DataProvider;
import org.apache.marmotta.ldclient.exception.DataRetrievalException;
import org.apache.marmotta.ldclient.model.ClientResponse;
+import org.openrdf.model.Model;
import org.openrdf.model.URI;
import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.TreeModel;
+import org.openrdf.model.impl.ValueFactoryImpl;
import org.openrdf.model.vocabulary.RDF;
import org.openrdf.model.vocabulary.SKOS;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepository;
-import org.openrdf.sail.memory.MemoryStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* A provider that accesses objects exposed by the Facebook Graph API (in JSON format). The provider will map the
@@ -187,94 +180,82 @@ public class FacebookGraphProvider implements DataProvider {
* retrieve for completing the data of the resource
*
* @param resourceUri
- * @param repository an RDF repository for storing an RDF representation of the dataset located at the remote resource.
+ * @param model an RDF model for storing an RDF representation of the dataset located at the remote resource.
* @param in input stream as returned by the remote webservice
* @param language content language as returned in the HTTP headers of the remote webservice
* @return a possibly empty list of URLs of additional resources to retrieve to complete the content
* @throws java.io.IOException in case an error occurs while reading the input stream
*/
- protected List<String> parseResponse(String resourceUri, String requestUrl, Repository repository, InputStream in, String language) throws DataRetrievalException {
+ protected List<String> parseResponse(String resourceUri, String requestUrl, Model model, InputStream in, String language) throws DataRetrievalException {
ObjectMapper mapper = new ObjectMapper();
try {
Map<String,Object> data = mapper.readValue(in, new TypeReference<Map<String,Object>>() { });
- RepositoryConnection con = repository.getConnection();
- try {
- con.begin();
+ ValueFactory vf = ValueFactoryImpl.getInstance();
- ValueFactory vf = repository.getValueFactory();
+ URI subject = vf.createURI(resourceUri);
- URI subject = vf.createURI(resourceUri);
-
- // add the type based on the facebook category
- if(data.get("category") != null) {
- con.add(subject, RDF.TYPE, getType(data.get("category").toString()));
- }
+ // add the type based on the facebook category
+ if(data.get("category") != null) {
+ model.add(subject, RDF.TYPE, getType(data.get("category").toString()));
+ }
- con.add(subject,DCTERMS.identifier,vf.createLiteral(data.get("id").toString()));
+ model.add(subject,DCTERMS.identifier,vf.createLiteral(data.get("id").toString()));
- // schema:name is the facebook name (can have multiple languages)
- con.add(subject, SCHEMA.name, vf.createLiteral(data.get("name").toString(), language));
- con.add(subject, DCTERMS.title, vf.createLiteral(data.get("name").toString(), language));
+ // schema:name is the facebook name (can have multiple languages)
+ model.add(subject, SCHEMA.name, vf.createLiteral(data.get("name").toString(), language));
+ model.add(subject, DCTERMS.title, vf.createLiteral(data.get("name").toString(), language));
- // dct:description in case a description or about is present (all content in English)
- if(data.get("description") != null) {
- con.add(subject,SCHEMA.description, vf.createLiteral(data.get("description").toString(), "en"));
- con.add(subject,DCTERMS.description, vf.createLiteral(data.get("description").toString(), "en"));
- }
- if(data.get("about") != null) {
- con.add(subject,SCHEMA.description, vf.createLiteral(data.get("about").toString(), "en"));
- con.add(subject,DCTERMS.description, vf.createLiteral(data.get("about").toString(), "en"));
- }
-
- // if there is genre information, add it using schema:genre and dct:subject
- if(data.get("genre") != null) {
- con.add(subject,SCHEMA.genre, vf.createLiteral(data.get("genre").toString()));
- con.add(subject,DCTERMS.subject, vf.createLiteral(data.get("genre").toString()));
- }
- if(data.get("directed_by") != null) {
- con.add(subject,SCHEMA.director, vf.createLiteral(data.get("directed_by").toString()));
- con.add(subject,DCTERMS.creator, vf.createLiteral(data.get("directed_by").toString()));
- }
- if(data.get("studio") != null) {
- con.add(subject,SCHEMA.publisher, vf.createLiteral(data.get("studio").toString()));
- con.add(subject,DCTERMS.publisher, vf.createLiteral(data.get("studio").toString()));
- }
- if(data.get("plot_outline") != null) {
- con.add(subject,SCHEMA.description, vf.createLiteral(data.get("plot_outline").toString()));
- con.add(subject,DCTERMS.description, vf.createLiteral(data.get("plot_outline").toString()));
- }
- if(data.get("phone") != null) {
- con.add(subject,SCHEMA.telephone, vf.createLiteral(data.get("phone").toString()));
- con.add(subject,FOAF.phone, vf.createLiteral(data.get("phone").toString()));
- }
- if(data.get("username") != null) {
- con.add(subject,FOAF.nick, vf.createLiteral(data.get("username").toString()));
- }
+ // dct:description in case a description or about is present (all content in English)
+ if(data.get("description") != null) {
+ model.add(subject,SCHEMA.description, vf.createLiteral(data.get("description").toString(), "en"));
+ model.add(subject,DCTERMS.description, vf.createLiteral(data.get("description").toString(), "en"));
+ }
+ if(data.get("about") != null) {
+ model.add(subject,SCHEMA.description, vf.createLiteral(data.get("about").toString(), "en"));
+ model.add(subject,DCTERMS.description, vf.createLiteral(data.get("about").toString(), "en"));
+ }
- if(data.get("cover") != null && data.get("cover") instanceof Map && ((Map<?,?>)data.get("cover")).get("source") != null) {
- con.add(subject,FOAF.thumbnail, vf.createURI(((Map<?,?>) data.get("cover")).get("source").toString()));
- }
+ // if there is genre information, add it using schema:genre and dct:subject
+ if(data.get("genre") != null) {
+ model.add(subject,SCHEMA.genre, vf.createLiteral(data.get("genre").toString()));
+ model.add(subject,DCTERMS.subject, vf.createLiteral(data.get("genre").toString()));
+ }
+ if(data.get("directed_by") != null) {
+ model.add(subject,SCHEMA.director, vf.createLiteral(data.get("directed_by").toString()));
+ model.add(subject,DCTERMS.creator, vf.createLiteral(data.get("directed_by").toString()));
+ }
+ if(data.get("studio") != null) {
+ model.add(subject,SCHEMA.publisher, vf.createLiteral(data.get("studio").toString()));
+ model.add(subject,DCTERMS.publisher, vf.createLiteral(data.get("studio").toString()));
+ }
+ if(data.get("plot_outline") != null) {
+ model.add(subject,SCHEMA.description, vf.createLiteral(data.get("plot_outline").toString()));
+ model.add(subject,DCTERMS.description, vf.createLiteral(data.get("plot_outline").toString()));
+ }
+ if(data.get("phone") != null) {
+ model.add(subject,SCHEMA.telephone, vf.createLiteral(data.get("phone").toString()));
+ model.add(subject,FOAF.phone, vf.createLiteral(data.get("phone").toString()));
+ }
+ if(data.get("username") != null) {
+ model.add(subject,FOAF.nick, vf.createLiteral(data.get("username").toString()));
+ }
+ if(data.get("cover") != null && data.get("cover") instanceof Map && ((Map<?,?>)data.get("cover")).get("source") != null) {
+ model.add(subject,FOAF.thumbnail, vf.createURI(((Map<?,?>) data.get("cover")).get("source").toString()));
+ }
- // website
- if(data.get("website") != null && UriUtil.validate(data.get("website").toString())) {
- con.add(subject, FOAF.homepage, vf.createURI(data.get("website").toString()));
- }
- if(data.get("link") != null) {
- con.add(subject, FOAF.homepage, vf.createURI(data.get("link").toString()));
- }
- con.commit();
- } catch(RepositoryException ex) {
- con.rollback();
- } finally {
- con.close();
+ // website
+ if(data.get("website") != null && UriUtil.validate(data.get("website").toString())) {
+ model.add(subject, FOAF.homepage, vf.createURI(data.get("website").toString()));
}
- } catch(RepositoryException ex) {
- log.error("error while storing retrieved triples in repository",ex);
+ if(data.get("link") != null) {
+ model.add(subject, FOAF.homepage, vf.createURI(data.get("link").toString()));
+ }
+
} catch (JsonMappingException e) {
throw new DataRetrievalException("error while mapping JSON response",e);
} catch (JsonParseException e) {
@@ -401,9 +382,7 @@ public class FacebookGraphProvider implements DataProvider {
}
if(log.isInfoEnabled()) {
- RepositoryConnection con = handler.triples.getConnection();
- log.info("retrieved {} triples for resource {}; expiry date: {}",new Object[] {con.size(),resourceUri,expiresDate});
- con.close();
+ log.info("retrieved {} triples for resource {}; expiry date: {}", new Object[]{handler.triples.size(), resourceUri, expiresDate});
}
ClientResponse result = new ClientResponse(200, handler.triples);
@@ -433,7 +412,7 @@ public class FacebookGraphProvider implements DataProvider {
private String requestUrl;
// the repository where the triples will be stored in case the data providers return them
- private final Repository triples;
+ private final Model triples;
private final Endpoint endpoint;
@@ -448,8 +427,7 @@ public class FacebookGraphProvider implements DataProvider {
this.resource = resource;
this.endpoint = endpoint;
- triples = new SailRepository(new MemoryStore());
- triples.initialize();
+ triples = new TreeModel();
}
@Override
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-provider-facebook/src/test/java/org/apache/marmotta/ldclient/test/facebook/FacebookProviderTest.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-provider-facebook/src/test/java/org/apache/marmotta/ldclient/test/facebook/FacebookProviderTest.java b/libraries/ldclient/ldclient-provider-facebook/src/test/java/org/apache/marmotta/ldclient/test/facebook/FacebookProviderTest.java
index 350c04b..481a5ae 100644
--- a/libraries/ldclient/ldclient-provider-facebook/src/test/java/org/apache/marmotta/ldclient/test/facebook/FacebookProviderTest.java
+++ b/libraries/ldclient/ldclient-provider-facebook/src/test/java/org/apache/marmotta/ldclient/test/facebook/FacebookProviderTest.java
@@ -17,146 +17,29 @@
package org.apache.marmotta.ldclient.test.facebook;
-import org.apache.commons.io.IOUtils;
-import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
-import org.apache.marmotta.ldclient.model.ClientResponse;
-import org.apache.marmotta.ldclient.services.ldclient.LDClient;
-import org.apache.marmotta.ldclient.test.helper.TestLDClient;
-import org.junit.*;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-import org.openrdf.query.BooleanQuery;
-import org.openrdf.query.QueryLanguage;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.rio.RDFFormat;
-import org.openrdf.rio.Rio;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.io.StringWriter;
+import org.apache.marmotta.ldclient.test.provider.ProviderTestBase;
+import org.junit.Test;
/**
* Test the Facebook Provider for some sample resources
* <p/>
* Author: Sebastian Schaffert (sschaffert@apache.org)
*/
-public class FacebookProviderTest {
-
- private LDClientService ldclient;
-
- private static Logger log = LoggerFactory.getLogger(FacebookProviderTest.class);
-
- @Before
- public void setupClient() {
- ldclient = new TestLDClient(new LDClient());
- }
-
- @After
- public void shutdownClient() {
- ldclient.shutdown();
- }
-
- final Logger logger =
- LoggerFactory.getLogger(this.getClass());
-
- @Rule
- public TestWatcher watchman = new TestWatcher() {
- /**
- * Invoked when a test is about to start
- */
- @Override
- protected void starting(Description description) {
- logger.info("{} being run...", description.getMethodName());
- }
- };
+public class FacebookProviderTest extends ProviderTestBase {
@Test
public void testMovie() throws Exception {
-
- String uriMovie = "https://graph.facebook.com/160617097307237";
-
- Assume.assumeTrue(ldclient.ping(uriMovie));
-
- ClientResponse respMovie = ldclient.retrieveResource(uriMovie);
-
- RepositoryConnection con = respMovie.getTriples().getConnection();
- con.begin();
- Assert.assertTrue(con.size() > 0);
-
- logData(con);
-
- // run a SPARQL test to see if the returned data is correct
- InputStream sparql = this.getClass().getResourceAsStream("movie.sparql");
- BooleanQuery testLabel = con.prepareBooleanQuery(QueryLanguage.SPARQL, IOUtils.toString(sparql));
- Assert.assertTrue("SPARQL test query failed", testLabel.evaluate());
-
-
- con.commit();
- con.close();
-
-
+ testResource("https://graph.facebook.com/160617097307237", "movie.sparql");
}
@Test
public void testInterest() throws Exception {
- String uriInterest = "http://graph.facebook.com/106515832719603";
-
- Assume.assumeTrue(ldclient.ping(uriInterest));
-
- ClientResponse respInterest = ldclient.retrieveResource(uriInterest);
-
- RepositoryConnection con = respInterest.getTriples().getConnection();
- con.begin();
- Assert.assertTrue(con.size() > 0);
-
- logData(con);
-
- // run a SPARQL test to see if the returned data is correct
- InputStream sparql = this.getClass().getResourceAsStream("interest.sparql");
- BooleanQuery testLabel = con.prepareBooleanQuery(QueryLanguage.SPARQL, IOUtils.toString(sparql));
- Assert.assertTrue("SPARQL test query failed", testLabel.evaluate());
-
-
- con.commit();
- con.close();
-
+ testResource("http://graph.facebook.com/106515832719603", "interest.sparql");
}
@Test
public void testRestauraunt() throws Exception {
- String uriRestaurant = "http://graph.facebook.com/285699076901";
-
- Assume.assumeTrue(ldclient.ping(uriRestaurant));
-
- ClientResponse respInterest = ldclient.retrieveResource(uriRestaurant);
-
- RepositoryConnection con = respInterest.getTriples().getConnection();
- con.begin();
- Assert.assertTrue(con.size() > 0);
-
- logData(con);
-
- // run a SPARQL test to see if the returned data is correct
- InputStream sparql = this.getClass().getResourceAsStream("restaurant.sparql");
- BooleanQuery testLabel = con.prepareBooleanQuery(QueryLanguage.SPARQL, IOUtils.toString(sparql));
- Assert.assertTrue("SPARQL test query failed", testLabel.evaluate());
-
-
- con.commit();
- con.close();
-
- }
-
-
- private void logData(RepositoryConnection con) throws Exception {
- if(log.isDebugEnabled()) {
- StringWriter out = new StringWriter();
- con.export(Rio.createWriter(RDFFormat.TURTLE, out));
- log.debug("DATA:");
- log.debug(out.toString());
- }
-
+ testResource("http://graph.facebook.com/285699076901", "restaurant.sparql");
}
}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-provider-html/src/main/java/org/apache/marmotta/ldclient/provider/html/AbstractHTMLDataProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-provider-html/src/main/java/org/apache/marmotta/ldclient/provider/html/AbstractHTMLDataProvider.java b/libraries/ldclient/ldclient-provider-html/src/main/java/org/apache/marmotta/ldclient/provider/html/AbstractHTMLDataProvider.java
index 1a54c70..bab5cf7 100644
--- a/libraries/ldclient/ldclient-provider-html/src/main/java/org/apache/marmotta/ldclient/provider/html/AbstractHTMLDataProvider.java
+++ b/libraries/ldclient/ldclient-provider-html/src/main/java/org/apache/marmotta/ldclient/provider/html/AbstractHTMLDataProvider.java
@@ -26,14 +26,8 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
-import org.openrdf.model.Resource;
-import org.openrdf.model.Statement;
-import org.openrdf.model.URI;
-import org.openrdf.model.Value;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.RepositoryException;
+import org.openrdf.model.*;
+import org.openrdf.model.impl.ValueFactoryImpl;
import java.io.IOException;
import java.io.InputStream;
@@ -83,14 +77,15 @@ public abstract class AbstractHTMLDataProvider extends AbstractHttpProvider impl
* decide how to parse the data.
*
*
+ *
* @param resource the subject of the data retrieval
- * @param in input stream as returned by the remote webservice
- * @param contentType content type as returned in the HTTP headers of the remote webservice
- * @return an RDF repository containing an RDF representation of the dataset located at the remote resource.
+ * @param triples
+ *@param in input stream as returned by the remote webservice
+ * @param contentType content type as returned in the HTTP headers of the remote webservice @return an RDF repository containing an RDF representation of the dataset located at the remote resource.
* @throws java.io.IOException in case an error occurs while reading the input stream
*/
@Override
- public List<String> parseResponse(String resource, String requestUrl, Repository triples, InputStream in, String contentType) throws DataRetrievalException {
+ public List<String> parseResponse(String resource, String requestUrl, Model triples, InputStream in, String contentType) throws DataRetrievalException {
String charset = null;
Pattern pattern = Pattern.compile("charset=([a-zA-Z0-9-_]+)");
Matcher matcher = pattern.matcher(contentType);
@@ -101,8 +96,7 @@ public abstract class AbstractHTMLDataProvider extends AbstractHttpProvider impl
try {
Document htmlDoc = Jsoup.parse(in,charset,requestUrl);
- RepositoryConnection con = triples.getConnection();
- ValueFactory vf = con.getValueFactory();
+ ValueFactory vf = ValueFactoryImpl.getInstance();
URI subject = vf.createURI(resource);
for (Map.Entry<String, JSoupMapper> mapping : getMappings(resource, requestUrl).entrySet()) {
@@ -113,7 +107,7 @@ public abstract class AbstractHTMLDataProvider extends AbstractHttpProvider impl
List<Value> objects = mapping.getValue().map(resource, value, vf);
for(Value object : objects) {
Statement stmt = vf.createStatement(subject, predicate, object);
- con.add(stmt);
+ triples.add(stmt);
}
}
}
@@ -122,16 +116,12 @@ public abstract class AbstractHTMLDataProvider extends AbstractHttpProvider impl
for(String typeUri : getTypes(subject)) {
Resource type_resource = vf.createURI(typeUri);
- con.add(vf.createStatement(subject, ptype, type_resource));
+ triples.add(vf.createStatement(subject, ptype, type_resource));
}
- con.commit();
- con.close();
return findAdditionalRequestUrls(resource, htmlDoc, requestUrl);
- } catch (RepositoryException e) {
- throw new DataRetrievalException("repository error while parsing XML response",e);
} catch (IOException e) {
throw new DataRetrievalException("I/O error while parsing HTML response",e);
}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/d10a5c60/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java b/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java
index 3873391..8797954 100644
--- a/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java
+++ b/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java
@@ -17,16 +17,6 @@
*/
package org.apache.marmotta.ldclient.provider.ldap;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.directory.api.ldap.model.cursor.CursorException;
import org.apache.directory.api.ldap.model.cursor.EntryCursor;
import org.apache.directory.api.ldap.model.entry.Attribute;
@@ -46,27 +36,29 @@ import org.apache.marmotta.ldclient.model.ClientResponse;
import org.apache.marmotta.ldclient.provider.ldap.mapping.LiteralPredicateFactory;
import org.apache.marmotta.ldclient.provider.ldap.mapping.PredicateObjectFactory;
import org.apache.marmotta.ldclient.provider.ldap.mapping.UriPredicateFactory;
+import org.openrdf.model.Model;
import org.openrdf.model.URI;
import org.openrdf.model.Value;
import org.openrdf.model.ValueFactory;
-import org.openrdf.repository.Repository;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepository;
-import org.openrdf.sail.memory.MemoryStore;
+import org.openrdf.model.impl.TreeModel;
+import org.openrdf.model.impl.ValueFactoryImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.*;
+
/**
* LdapFoafProvider maps LDAP accounts to foaf:Person instances
- *
+ *
* @author Daniel Trabe <da...@salzburgresearch.at>
* @author Jakob Frank <ja...@salzburgresearch.at>
* @author Sergio Fernández <wi...@apache.org>
- *
+ *
*/
public class LdapFoafProvider implements DataProvider {
-
+
public static final String PROVIDER_NAME = "LDAP";
private static Logger log = LoggerFactory.getLogger(LdapFoafProvider.class);
@@ -114,38 +106,37 @@ public class LdapFoafProvider implements DataProvider {
* Opens a connection tho the LDAP-server. in most cases an Account + Password is needed to
* connect
* to the Server. This need to be configured at the endpoint
- *
+ *
* @param endpoint
* @return
- * @throws LDAPException if connecting failed.
* @throws DataRetrievalException
*/
private LdapConnection openLdapConnection(Endpoint endpoint) throws DataRetrievalException {
java.net.URI u;
- try {
- u = new java.net.URI(endpoint.getEndpointUrl());
- } catch (URISyntaxException e) {
- throw new DataRetrievalException("Invalid enpooint URI", e);
- }
- LdapConnectionConfig config = new LdapConnectionConfig();
- config.setLdapHost(u.getHost());
- config.setLdapPort(u.getPort() > 0 ? u.getPort() : 389);
- if (endpoint.hasProperty("loginDN")) {
- config.setName(endpoint.getProperty("loginDN"));
- }
- if (endpoint.hasProperty("loginPW")) {
- config.setName(endpoint.getProperty("loginPW"));
- }
+ try {
+ u = new java.net.URI(endpoint.getEndpointUrl());
+ } catch (URISyntaxException e) {
+ throw new DataRetrievalException("Invalid enpooint URI", e);
+ }
+ LdapConnectionConfig config = new LdapConnectionConfig();
+ config.setLdapHost(u.getHost());
+ config.setLdapPort(u.getPort() > 0 ? u.getPort() : 389);
+ if (endpoint.hasProperty("loginDN")) {
+ config.setName(endpoint.getProperty("loginDN"));
+ }
+ if (endpoint.hasProperty("loginPW")) {
+ config.setName(endpoint.getProperty("loginPW"));
+ }
LdapNetworkConnection connection = new LdapNetworkConnection(config);
try {
- connection.bind();
- } catch (Exception e) {
- throw new DataRetrievalException("LDAP connnection could not be bind", e);
- }
- if (connection.isAuthenticated()) {
- return connection;
+ connection.bind();
+ } catch (Exception e) {
+ throw new DataRetrievalException("LDAP connnection could not be bind", e);
+ }
+ if (connection.isAuthenticated()) {
+ return connection;
} else {
- throw new DataRetrievalException("LDAP connnection could not be stablished");
+ throw new DataRetrievalException("LDAP connnection could not be stablished");
}
}
@@ -157,45 +148,36 @@ public class LdapFoafProvider implements DataProvider {
try {
final LdapConnection ldap = openLdapConnection(endpoint);
- Repository rep = new SailRepository(new MemoryStore());
- rep.initialize();
-
- RepositoryConnection conn = rep.getConnection();
- try {
- ValueFactory vf = conn.getValueFactory();
- String userDN = buildDN(prefix, account, ldap);
+ Model model = new TreeModel();
+ ValueFactory vf = ValueFactoryImpl.getInstance();
+ String userDN = buildDN(prefix, account, ldap);
- Map<String, java.util.List<String>> accountData = getAccountData(userDN, ldap);
+ Map<String, java.util.List<String>> accountData = getAccountData(userDN, ldap);
- final URI subject = vf.createURI(resource);
- for (String attr : MAPPING.keySet()) {
- if (!accountData.containsKey(attr)) {
- continue;
- }
+ final URI subject = vf.createURI(resource);
+ for (String attr : MAPPING.keySet()) {
+ if (!accountData.containsKey(attr)) {
+ continue;
+ }
- final PredicateObjectFactory factory = MAPPING.get(attr);
- final URI predicate = factory.createPredicate(vf);
+ final PredicateObjectFactory factory = MAPPING.get(attr);
+ final URI predicate = factory.createPredicate(vf);
- for (String val : accountData.get(attr)) {
- for (Value object : factory.createObjects(val, vf)) {
- conn.add(vf.createStatement(subject, predicate, object));
- }
+ for (String val : accountData.get(attr)) {
+ for (Value object : factory.createObjects(val, vf)) {
+ model.add(vf.createStatement(subject, predicate, object));
}
-
}
- final ClientResponse resp = new ClientResponse(200, rep);
- resp.setExpires(new Date());
- return resp;
-
- } catch (Exception e) {
- throw new DataRetrievalException(e);
- } finally {
- conn.close();
}
- } catch (RepositoryException e1) {
- log.warn("Could not create SailRep: {}", e1.getMessage());
- throw new DataRetrievalException(e1);
+
+ final ClientResponse resp = new ClientResponse(200, model);
+ resp.setExpires(new Date());
+ return resp;
+
+ } catch (LdapException | CursorException | IOException e) {
+ log.warn("could not access LDAP repository: {}", e.getMessage());
+ throw new DataRetrievalException(e);
}
}
@@ -203,7 +185,7 @@ public class LdapFoafProvider implements DataProvider {
* Returns the suffix (e.g. "dc=salzburgresearch,dc=at" of the Endpoint. This suffix is needed
* to perform
* miscellaneous LDAP-Operations
- *
+ *
* @param endpoint
* @return
* @throws DataRetrievalException when Config of LDAP-Suffix for endpoint is invalid
@@ -223,12 +205,11 @@ public class LdapFoafProvider implements DataProvider {
* on the URL we get from the resource.
* (e.g "/SRFG/USERS/Daniel%20Trabe" ==>
* "cn=Daniel Trabe,ou=USERS,ou=SRFG,dc=salzburgresearch,dc=at")
- *
+ *
* @param suffix the LDAP-Suffix configured at the Endpoint
* @param path part of the URL (e.g. /SRFG/USERS/DanielTrabe)
* @param con an initialized LDAP-Context
* @return the distinguished name of an Entry in the LDAP
- * @throws LDAPException if there is a problem with the LDAP connection
* @throws DataRetrievalException when the Entry cannot be found
*/
private String buildDN(String suffix, String path, LdapConnection con) throws DataRetrievalException {
@@ -237,12 +218,12 @@ public class LdapFoafProvider implements DataProvider {
String current = path.split("/")[0];
List<String> data;
- try {
- data = getChildList(suffix, con);
- } catch (Exception e) {
- log.error("Error getting childs: {}", e.getMessage(), e);
- throw new DataRetrievalException(e);
- }
+ try {
+ data = getChildList(suffix, con);
+ } catch (Exception e) {
+ log.error("Error getting childs: {}", e.getMessage(), e);
+ throw new DataRetrievalException(e);
+ }
String next = null;
for (String dn : data) {
if (dn.toLowerCase().endsWith((current + "," + suffix).toLowerCase())) {
@@ -258,14 +239,13 @@ public class LdapFoafProvider implements DataProvider {
/**
* Get child list
- *
+ *
* @param entryDN The distinguished name of an Entry in the LDAP
- * @param con An initialized LDAP-Context
+ * @param connection An initialized LDAP-Context
* @return All child's of an Entry
- * @throws IOException
- * @throws CursorException
- * @throws LdapException
- * @throws LDAPSearchException
+ * @throws IOException
+ * @throws CursorException
+ * @throws LdapException
*/
private List<String> getChildList(String entryDN, LdapConnection connection) throws CursorException, IOException, LdapException {
List<String> childs = new ArrayList<String>();
@@ -285,31 +265,30 @@ public class LdapFoafProvider implements DataProvider {
/**
* Get account data
- *
+ *
* @param accountDN The distinguished Name of the Account (e.g
* "cn=Daniel Trabe,ou=USERS,ou=SRFG,dc=salzburgresearch,dc=at")
* @param connection An initialized LDAP-Context
* @return a Map of Attributes and Values of an Account
* @throws DataRetrievalException
- * @throws IOException
- * @throws CursorException
- * @throws LdapException
- * @throws LDAPException
+ * @throws IOException
+ * @throws CursorException
+ * @throws LdapException
*/
private Map<String, List<String>> getAccountData(String accountDN, LdapConnection connection) throws DataRetrievalException, LdapException, CursorException, IOException {
Map<String, List<String>> account = new HashMap<String, List<String>>();
Dn dn = new Dn(accountDN);
EntryCursor cursor = connection.search(dn, accountDN, SearchScope.ONELEVEL, (String[])null);
if (cursor.next()) {
- //FIXME: only the first entry?
+ //FIXME: only the first entry?
Entry entry = cursor.get();
for (Attribute attr : entry.getAttributes()) {
String id = attr.getId();
- List<String> values;
+ List<String> values;
if (account.containsKey(id)) {
- values = account.get(id);
+ values = account.get(id);
} else {
- values = new ArrayList<String>();
+ values = new ArrayList<String>();
}
values.add(attr.get().getValue().toString());
account.put(id, values);