You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@any23.apache.org by mo...@apache.org on 2012/05/17 21:16:50 UTC
svn commit: r1339808 - in /incubator/any23/trunk/core/src:
main/java/org/apache/any23/cli/MicrodataParser.java
main/java/org/apache/any23/http/DefaultHTTPClient.java
test/java/org/apache/any23/cli/MicrodataParserTest.java
Author: mostarda
Date: Thu May 17 19:16:49 2012
New Revision: 1339808
URL: http://svn.apache.org/viewvc?rev=1339808&view=rev
Log:
Fixed MicrodataParser CLI issue with HTTP resources. Added specific test case, improved usability of DefaultHTTPClient. Related to issue #ANY23-94.
Modified:
incubator/any23/trunk/core/src/main/java/org/apache/any23/cli/MicrodataParser.java
incubator/any23/trunk/core/src/main/java/org/apache/any23/http/DefaultHTTPClient.java
incubator/any23/trunk/core/src/test/java/org/apache/any23/cli/MicrodataParserTest.java
Modified: incubator/any23/trunk/core/src/main/java/org/apache/any23/cli/MicrodataParser.java
URL: http://svn.apache.org/viewvc/incubator/any23/trunk/core/src/main/java/org/apache/any23/cli/MicrodataParser.java?rev=1339808&r1=1339807&r2=1339808&view=diff
==============================================================================
--- incubator/any23/trunk/core/src/main/java/org/apache/any23/cli/MicrodataParser.java (original)
+++ incubator/any23/trunk/core/src/main/java/org/apache/any23/cli/MicrodataParser.java Thu May 17 19:16:49 2012
@@ -84,7 +84,7 @@ public class MicrodataParser implements
final Matcher httpMatcher = HTTP_DOCUMENT_PATTERN.matcher(value);
if (httpMatcher.find()) {
try {
- return new HTTPDocumentSource(new DefaultHTTPClient(), value);
+ return new HTTPDocumentSource(DefaultHTTPClient.createInitializedHTTPClient(), value);
} catch ( URISyntaxException e ) {
throw new ParameterException("Invalid source URI: '" + value + "'");
}
Modified: incubator/any23/trunk/core/src/main/java/org/apache/any23/http/DefaultHTTPClient.java
URL: http://svn.apache.org/viewvc/incubator/any23/trunk/core/src/main/java/org/apache/any23/http/DefaultHTTPClient.java?rev=1339808&r1=1339807&r2=1339808&view=diff
==============================================================================
--- incubator/any23/trunk/core/src/main/java/org/apache/any23/http/DefaultHTTPClient.java (original)
+++ incubator/any23/trunk/core/src/main/java/org/apache/any23/http/DefaultHTTPClient.java Thu May 17 19:16:49 2012
@@ -17,6 +17,7 @@
package org.apache.any23.http;
+import org.apache.any23.configuration.DefaultConfiguration;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
@@ -55,6 +56,42 @@ public class DefaultHTTPClient implement
private String contentType = null;
+ /**
+ * Creates a default {@link HTTPClientConfiguration} instance.
+ *
+ * @return a deault configuration.
+ */
+ public static HTTPClientConfiguration createDefaultConfiguration() {
+ return new HTTPClientConfiguration() {
+ public String getUserAgent() {
+ return DefaultConfiguration.singleton().getPropertyOrFail("any23.http.user.agent.default");
+ }
+
+ public String getAcceptHeader() {
+ return null;
+ }
+
+ public int getDefaultTimeout() {
+ return DefaultConfiguration.singleton().getPropertyIntOrFail("any23.http.client.timeout");
+ }
+
+ public int getMaxConnections() {
+ return DefaultConfiguration.singleton().getPropertyIntOrFail("any23.http.client.max.connections");
+ }
+ };
+ }
+
+ /**
+ * Creates a {@link DefaultHTTPClient} instance already initialized
+ *
+ * @return
+ */
+ public static DefaultHTTPClient createInitializedHTTPClient() {
+ final DefaultHTTPClient defaultHTTPClient = new DefaultHTTPClient();
+ defaultHTTPClient.init( createDefaultConfiguration() );
+ return defaultHTTPClient;
+ }
+
public void init(HTTPClientConfiguration configuration) {
if(configuration == null) throw new NullPointerException("Illegal configuration, cannot be null.");
this.configuration = configuration;
@@ -73,7 +110,7 @@ public class DefaultHTTPClient implement
GetMethod method = null;
try {
ensureClientInitialized();
- String uriStr = null;
+ String uriStr;
try {
URI uriObj = new URI(uri);
// [scheme:][//authority][path][?query][#fragment]
@@ -162,7 +199,7 @@ public class DefaultHTTPClient implement
if (configuration.getAcceptHeader() != null) {
headers.add(new Header("Accept", configuration.getAcceptHeader()));
}
- headers.add(new Header("Accept-Language", "en-us,en-gb,en,*;q=0.3"));
+ headers.add(new Header("Accept-Language", "en-us,en-gb,en,*;q=0.3")); //TODO: this must become parametric.
headers.add(new Header("Accept-Charset", "utf-8,iso-8859-1;q=0.7,*;q=0.5"));
// headers.add(new Header("Accept-Encoding", "x-gzip, gzip"));
hostConf.getParams().setParameter("http.default-headers", headers);
Modified: incubator/any23/trunk/core/src/test/java/org/apache/any23/cli/MicrodataParserTest.java
URL: http://svn.apache.org/viewvc/incubator/any23/trunk/core/src/test/java/org/apache/any23/cli/MicrodataParserTest.java?rev=1339808&r1=1339807&r2=1339808&view=diff
==============================================================================
--- incubator/any23/trunk/core/src/test/java/org/apache/any23/cli/MicrodataParserTest.java (original)
+++ incubator/any23/trunk/core/src/test/java/org/apache/any23/cli/MicrodataParserTest.java Thu May 17 19:16:49 2012
@@ -31,8 +31,13 @@ public class MicrodataParserTest extends
}
@Test
- public void testRun() throws Exception {
+ public void testRunOnFile() throws Exception {
runToolCheckExit0("file:src/test/resources/microdata/microdata-nested.html");
}
+ @Test
+ public void testRunOnHTTPResource() throws Exception {
+ runToolCheckExit0("http://www.imdb.com/title/tt1375666/");
+ }
+
}