You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/07/17 17:39:19 UTC
git commit: LDClient: implement retry after for temporary failures of
servers and set a lower default value in case the header is not given (fixes
MARMOTTA-161 and MARMOTTA-163)
Updated Branches:
refs/heads/develop dd5db497e -> 64ba0285a
LDClient: implement retry after for temporary failures of servers and set a lower default value in case the header is not given (fixes MARMOTTA-161 and MARMOTTA-163)
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/64ba0285
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/64ba0285
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/64ba0285
Branch: refs/heads/develop
Commit: 64ba0285a5d0fa0c0240cbf86071959968f7c087
Parents: dd5db49
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Wed Jul 17 17:39:13 2013 +0200
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Wed Jul 17 17:39:13 2013 +0200
----------------------------------------------------------------------
.../services/test/dummy/DummyProvider.java | 2 +-
.../ldcache/sail/test/dummy/DummyProvider.java | 2 +-
.../ldcache/sail/test/dummy/DummyProvider.java | 2 +-
.../marmotta/ldclient/model/ClientResponse.java | 15 ++++++++++-
.../services/provider/AbstractHttpProvider.java | 26 ++++++++++++++++---
.../services/provider/BlockingProvider.java | 2 +-
.../facebook/FacebookGraphProvider.java | 27 +++++++++++++++-----
.../provider/ldap/LdapFoafProvider.java | 2 +-
.../youtube/YoutubeVideoPagesProvider.java | 2 +-
9 files changed, 63 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldcache/ldcache-core/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-core/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java b/libraries/ldcache/ldcache-core/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
index ca1b451..4b6f92a 100644
--- a/libraries/ldcache/ldcache-core/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
+++ b/libraries/ldcache/ldcache-core/src/test/java/org/apache/marmotta/ldcache/services/test/dummy/DummyProvider.java
@@ -69,7 +69,7 @@ public class DummyProvider implements DataProvider {
con.close();
}
- ClientResponse response = new ClientResponse(triples);
+ ClientResponse response = new ClientResponse(200, triples);
return response;
} catch (RepositoryException e) {
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java b/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
index cd58e55..15f3d81 100644
--- a/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
+++ b/libraries/ldcache/ldcache-sail-generic/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
@@ -69,7 +69,7 @@ public class DummyProvider implements DataProvider {
con.close();
}
- ClientResponse response = new ClientResponse(triples);
+ ClientResponse response = new ClientResponse(200, triples);
return response;
} catch (RepositoryException e) {
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java b/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
index cd58e55..15f3d81 100644
--- a/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
+++ b/libraries/ldcache/ldcache-sail-kiwi/src/test/java/org/apache/marmotta/ldcache/sail/test/dummy/DummyProvider.java
@@ -69,7 +69,7 @@ public class DummyProvider implements DataProvider {
con.close();
}
- ClientResponse response = new ClientResponse(triples);
+ ClientResponse response = new ClientResponse(200, triples);
return response;
} catch (RepositoryException e) {
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java b/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java
index 2afafee..a34d4be 100644
--- a/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java
+++ b/libraries/ldclient/ldclient-api/src/main/java/org/apache/marmotta/ldclient/model/ClientResponse.java
@@ -32,13 +32,19 @@ public class ClientResponse {
private static final int DEFAULT_EXPIRATION_IN_DAYS = 7;
+ /**
+ * The HTTP status code returned by the client
+ */
+ private int httpStatus;
+
private Repository triples;
private Date expires;
- public ClientResponse(Repository triples) {
+ public ClientResponse(int httpStatus, Repository triples) {
this.triples = triples;
this.expires = DateUtils.addDays(new Date(), DEFAULT_EXPIRATION_IN_DAYS);
+ this.httpStatus = this.httpStatus;
}
public Repository getTriples() {
@@ -57,4 +63,11 @@ public class ClientResponse {
this.expires = new Date(expires.getTime());
}
+ public int getHttpStatus() {
+ return httpStatus;
+ }
+
+ public void setHttpStatus(int httpStatus) {
+ this.httpStatus = httpStatus;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java b/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java
index 175baa5..ef89751 100644
--- a/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java
+++ b/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/AbstractHttpProvider.java
@@ -60,6 +60,7 @@ import static org.apache.marmotta.commons.http.LMFHttpUtils.parseContentType;
*/
public abstract class AbstractHttpProvider implements DataProvider {
+ public static final int RETRY_AFTER = 60;
private static Logger log = LoggerFactory.getLogger(AbstractHttpProvider.class);
/**
@@ -175,7 +176,7 @@ public abstract class AbstractHttpProvider implements DataProvider {
con.close();
}
- ClientResponse result = new ClientResponse(handler.triples);
+ ClientResponse result = new ClientResponse(handler.httpStatus, handler.triples);
result.setExpires(expiresDate);
return result;
} catch (RepositoryException e) {
@@ -228,6 +229,8 @@ public abstract class AbstractHttpProvider implements DataProvider {
private final String resource;
+ private int httpStatus;
+
public ResponseHandler(String resource, Endpoint endpoint) throws RepositoryException {
this.resource = resource;
this.endpoint = endpoint;
@@ -244,13 +247,15 @@ public abstract class AbstractHttpProvider implements DataProvider {
final HttpEntity entity = response.getEntity();
if (entity == null)
throw new IOException("no content returned by Linked Data resource " + resource);
-
+
if (!isValidContentType(entity.getContentType().getValue().split(";")[0], endpoint)) {
// FIXME: here was get.abort()
throw new IOException("invalid content returned by Linked Data resource " + resource + ": "
+ entity.getContentType().getValue());
}
+ this.httpStatus = response.getStatusLine().getStatusCode();
+
if (entity != null) {
String parseContentType = "application/rdf+xml";
if (endpoint != null && "SPARQL".equals(endpoint.getType())) {
@@ -282,8 +287,23 @@ public abstract class AbstractHttpProvider implements DataProvider {
} finally {
in.close();
}
- }
+ }
EntityUtils.consume(entity);
+ } else if(response.getStatusLine().getStatusCode() == 500 || response.getStatusLine().getStatusCode() == 503 || response.getStatusLine().getStatusCode() == 504) {
+ this.httpStatus = response.getStatusLine().getStatusCode();
+
+ Header retry = response.getFirstHeader("Retry-After");
+ if(retry != null) {
+ try {
+ int duration = Integer.parseInt(retry.getValue());
+ expiresDate = new Date(System.currentTimeMillis() + duration*1000);
+ } catch(NumberFormatException ex) {
+ log.debug("error parsing Retry-After: header");
+ }
+ } else {
+ expiresDate = new Date(System.currentTimeMillis() + RETRY_AFTER *1000);
+ }
+
} else {
log.error("the HTTP request failed (status: {})", response.getStatusLine());
throw new ClientProtocolException("the HTTP request failed (status: " + response.getStatusLine() + ")");
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java b/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java
index feec3f4..ae75d59 100644
--- a/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java
+++ b/libraries/ldclient/ldclient-core/src/main/java/org/apache/marmotta/ldclient/services/provider/BlockingProvider.java
@@ -87,7 +87,7 @@ public class BlockingProvider implements DataProvider {
long defaultExpires = client.getClientConfiguration().getDefaultExpiry();
Date expiresDate = new Date(System.currentTimeMillis() + defaultExpires * 1000);
- ClientResponse result = new ClientResponse(empty_repository);
+ ClientResponse result = new ClientResponse(200, empty_repository);
result.setExpires(expiresDate);
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java b/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java
index c7fb128..e9d606a 100644
--- a/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java
+++ b/libraries/ldclient/ldclient-provider-facebook/src/main/java/org/apache/marmotta/ldclient/provider/facebook/FacebookGraphProvider.java
@@ -22,12 +22,9 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URIUtils;
import org.apache.http.impl.cookie.DateParseException;
import org.apache.http.impl.cookie.DateUtils;
import org.apache.http.util.EntityUtils;
-import org.apache.marmotta.commons.collections.CollectionUtils;
-import org.apache.marmotta.commons.http.ContentType;
import org.apache.marmotta.commons.http.UriUtil;
import org.apache.marmotta.commons.vocabulary.DCTERMS;
import org.apache.marmotta.commons.vocabulary.FOAF;
@@ -37,7 +34,6 @@ import org.apache.marmotta.ldclient.api.ldclient.LDClientService;
import org.apache.marmotta.ldclient.api.provider.DataProvider;
import org.apache.marmotta.ldclient.exception.DataRetrievalException;
import org.apache.marmotta.ldclient.model.ClientResponse;
-import org.apache.marmotta.ldclient.services.provider.AbstractHttpProvider;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
@@ -60,8 +56,6 @@ import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import static org.apache.marmotta.commons.http.LMFHttpUtils.parseContentType;
-
/**
* A provider that accesses objects exposed by the Facebook Graph API (in JSON format). The provider will map the
* properties of Facebook Objects to RDF and tries choosing the most appropriate RDF types and vocabularies. The base
@@ -76,6 +70,7 @@ public class FacebookGraphProvider implements DataProvider {
private static final Pattern pattern = Pattern.compile("http://www\\.facebook\\.com/([^/]+/)*([^/]+)");
+ public static final int RETRY_AFTER = 60;
private static Logger log = LoggerFactory.getLogger(FacebookGraphProvider.class);
@@ -406,7 +401,7 @@ public class FacebookGraphProvider implements DataProvider {
con.close();
}
- ClientResponse result = new ClientResponse(handler.triples);
+ ClientResponse result = new ClientResponse(200, handler.triples);
result.setExpires(expiresDate);
return result;
} catch (RepositoryException e) {
@@ -439,6 +434,8 @@ public class FacebookGraphProvider implements DataProvider {
private final String resource;
+ private int httpStatus;
+
// language tag to use for literals
public String language;
@@ -459,6 +456,7 @@ public class FacebookGraphProvider implements DataProvider {
if (entity == null)
throw new IOException("no content returned by Linked Data resource " + resource);
+ this.httpStatus = response.getStatusLine().getStatusCode();
if (entity != null) {
InputStream in = entity.getContent();
@@ -486,6 +484,21 @@ public class FacebookGraphProvider implements DataProvider {
}
}
EntityUtils.consume(entity);
+ } else if(response.getStatusLine().getStatusCode() == 500 || response.getStatusLine().getStatusCode() == 503 || response.getStatusLine().getStatusCode() == 504) {
+ this.httpStatus = response.getStatusLine().getStatusCode();
+
+ Header retry = response.getFirstHeader("Retry-After");
+ if(retry != null) {
+ try {
+ int duration = Integer.parseInt(retry.getValue());
+ expiresDate = new Date(System.currentTimeMillis() + duration*1000);
+ } catch(NumberFormatException ex) {
+ log.debug("error parsing Retry-After: header");
+ }
+ } else {
+ expiresDate = new Date(System.currentTimeMillis() + RETRY_AFTER *1000);
+ }
+
} else {
log.error("the HTTP request failed (status: {})", response.getStatusLine());
throw new ClientProtocolException("the HTTP request failed (status: " + response.getStatusLine() + ")");
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java b/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java
index e0ade1b..3873391 100644
--- a/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java
+++ b/libraries/ldclient/ldclient-provider-ldap/src/main/java/org/apache/marmotta/ldclient/provider/ldap/LdapFoafProvider.java
@@ -184,7 +184,7 @@ public class LdapFoafProvider implements DataProvider {
}
- final ClientResponse resp = new ClientResponse(rep);
+ final ClientResponse resp = new ClientResponse(200, rep);
resp.setExpires(new Date());
return resp;
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/64ba0285/libraries/ldclient/ldclient-provider-youtube/src/main/java/org/apache/marmotta/ldclient/provider/youtube/YoutubeVideoPagesProvider.java
----------------------------------------------------------------------
diff --git a/libraries/ldclient/ldclient-provider-youtube/src/main/java/org/apache/marmotta/ldclient/provider/youtube/YoutubeVideoPagesProvider.java b/libraries/ldclient/ldclient-provider-youtube/src/main/java/org/apache/marmotta/ldclient/provider/youtube/YoutubeVideoPagesProvider.java
index 5f1389b..7258510 100644
--- a/libraries/ldclient/ldclient-provider-youtube/src/main/java/org/apache/marmotta/ldclient/provider/youtube/YoutubeVideoPagesProvider.java
+++ b/libraries/ldclient/ldclient-provider-youtube/src/main/java/org/apache/marmotta/ldclient/provider/youtube/YoutubeVideoPagesProvider.java
@@ -122,7 +122,7 @@ public class YoutubeVideoPagesProvider implements DataProvider {
log.error(msg);
throw new RuntimeException(msg, e);
}
- ClientResponse clientResponse = new ClientResponse(triples);
+ ClientResponse clientResponse = new ClientResponse(200, triples);
clientResponse.setExpires(DateUtils.addYears(new Date(), 10));
return clientResponse;
}