You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/12/06 22:49:16 UTC
svn commit: r1418099 - in /manifoldcf/trunk: CHANGES.txt
connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
Author: kwright
Date: Thu Dec 6 21:49:15 2012
New Revision: 1418099
URL: http://svn.apache.org/viewvc?rev=1418099&view=rev
Log:
Fix for CONNECTORS-579. Add untrusted, unverified SSL support to the RSS connector.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1418099&r1=1418098&r2=1418099&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Thu Dec 6 21:49:15 2012
@@ -3,6 +3,10 @@ $Id$
======================= 1.1-dev =====================
+CONNECTORS-579: Add untrusted, unverified https support to the RSS
+connector.
+(David Morano, Karl Wright)
+
CONNECTORS-575: Modify Livelink connector build to use stub only when
there is no lapi.jar available. This is to support the fact that lapi jars are
not compatible from release to release, class-wise, so clients will need to
Modified: manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1418099&r1=1418098&r2=1418099&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java (original)
+++ manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java Thu Dec 6 21:49:15 2012
@@ -26,6 +26,7 @@ import org.apache.manifoldcf.crawler.sys
import org.apache.manifoldcf.crawler.system.ManifoldCF;
import java.util.*;
import java.io.*;
+import java.net.*;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.client.HttpClient;
@@ -51,6 +52,7 @@ import org.apache.http.Header;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.message.BasicHeader;
import org.apache.http.client.params.ClientPNames;
+import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.client.CircularRedirectException;
@@ -264,10 +266,20 @@ public class ThrottledFetcher
this.minimumMillisecondsPerFetchPerServer = minimumMillisecondsPerFetchPerServer;
this.server = server;
this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
+
+ // Create the https scheme for this connection
+ javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();;
+ SSLSocketFactory myFactory = new SSLSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeoutMilliseconds),
+ new AllowAllHostnameVerifier());
+ Scheme myHttpsProtocol = new Scheme("https", 443, myFactory);
+
PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
localConnectionManager.setMaxTotal(1);
connectionManager = localConnectionManager;
-
+
+ // Set up protocol registry
+ connectionManager.getSchemeRegistry().register(myHttpsProtocol);
+
BasicHttpParams params = new BasicHttpParams();
params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
@@ -1427,4 +1439,176 @@ public class ThrottledFetcher
}
+ /** SSL Socket factory which wraps another socket factory but allows timeout on socket
+ * creation.
+ */
+ protected static class InterruptibleSocketFactory extends javax.net.ssl.SSLSocketFactory
+ {
+ protected final javax.net.ssl.SSLSocketFactory wrappedFactory;
+ protected final long connectTimeoutMilliseconds;
+
+ public InterruptibleSocketFactory(javax.net.ssl.SSLSocketFactory wrappedFactory, long connectTimeoutMilliseconds)
+ {
+ this.wrappedFactory = wrappedFactory;
+ this.connectTimeoutMilliseconds = connectTimeoutMilliseconds;
+ }
+
+ @Override
+ public Socket createSocket()
+ throws IOException
+ {
+ // Socket isn't open
+ return wrappedFactory.createSocket();
+ }
+
+ @Override
+ public Socket createSocket(String host, int port)
+ throws IOException, UnknownHostException
+ {
+ return fireOffThread(InetAddress.getByName(host),port,null,-1);
+ }
+
+ @Override
+ public Socket createSocket(InetAddress host, int port)
+ throws IOException
+ {
+ return fireOffThread(host,port,null,-1);
+ }
+
+ @Override
+ public Socket createSocket(String host, int port, InetAddress localHost, int localPort)
+ throws IOException, UnknownHostException
+ {
+ return fireOffThread(InetAddress.getByName(host),port,localHost,localPort);
+ }
+
+ @Override
+ public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort)
+ throws IOException
+ {
+ return fireOffThread(address,port,localAddress,localPort);
+ }
+
+ @Override
+ public Socket createSocket(Socket s, String host, int port, boolean autoClose)
+ throws IOException
+ {
+ // Socket's already open
+ return wrappedFactory.createSocket(s,host,port,autoClose);
+ }
+
+ @Override
+ public String[] getDefaultCipherSuites()
+ {
+ return wrappedFactory.getDefaultCipherSuites();
+ }
+
+ @Override
+ public String[] getSupportedCipherSuites()
+ {
+ return wrappedFactory.getSupportedCipherSuites();
+ }
+
+ protected Socket fireOffThread(InetAddress address, int port, InetAddress localHost, int localPort)
+ throws IOException
+ {
+ SocketCreateThread thread = new SocketCreateThread(wrappedFactory,address,port,localHost,localPort);
+ thread.start();
+ try
+ {
+ // Wait for thread to complete for only a certain amount of time!
+ thread.join(connectTimeoutMilliseconds);
+ // If join() times out, then the thread is going to still be alive.
+ if (thread.isAlive())
+ {
+ // Kill the thread - not that this will necessarily work, but we need to try
+ thread.interrupt();
+ throw new ConnectTimeoutException("Secure connection timed out");
+ }
+ // The thread terminated. Throw an error if there is one, otherwise return the result.
+ Throwable t = thread.getException();
+ if (t != null)
+ {
+ if (t instanceof java.net.SocketTimeoutException)
+ throw (java.net.SocketTimeoutException)t;
+ else if (t instanceof ConnectTimeoutException)
+ throw (ConnectTimeoutException)t;
+ else if (t instanceof InterruptedIOException)
+ throw (InterruptedIOException)t;
+ else if (t instanceof IOException)
+ throw (IOException)t;
+ else if (t instanceof Error)
+ throw (Error)t;
+ else if (t instanceof RuntimeException)
+ throw (RuntimeException)t;
+ throw new Error("Received an unexpected exception: "+t.getMessage(),t);
+ }
+ return thread.getResult();
+ }
+ catch (InterruptedException e)
+ {
+ throw new InterruptedIOException("Interrupted: "+e.getMessage());
+ }
+
+ }
+
+ }
+
+ /** Create a secure socket in a thread, so that we can "give up" after a while if the socket fails to connect.
+ */
+ protected static class SocketCreateThread extends Thread
+ {
+ // Socket factory
+ protected javax.net.ssl.SSLSocketFactory socketFactory;
+ protected InetAddress host;
+ protected int port;
+ protected InetAddress clientHost;
+ protected int clientPort;
+
+ // The return socket
+ protected Socket rval = null;
+ // The return error
+ protected Throwable throwable = null;
+
+ /** Create the thread */
+ public SocketCreateThread(javax.net.ssl.SSLSocketFactory socketFactory,
+ InetAddress host,
+ int port,
+ InetAddress clientHost,
+ int clientPort)
+ {
+ this.socketFactory = socketFactory;
+ this.host = host;
+ this.port = port;
+ this.clientHost = clientHost;
+ this.clientPort = clientPort;
+ setDaemon(true);
+ }
+
+ public void run()
+ {
+ try
+ {
+ if (clientHost == null)
+ rval = socketFactory.createSocket(host,port);
+ else
+ rval = socketFactory.createSocket(host,port,clientHost,clientPort);
+ }
+ catch (Throwable e)
+ {
+ throwable = e;
+ }
+ }
+
+ public Throwable getException()
+ {
+ return throwable;
+ }
+
+ public Socket getResult()
+ {
+ return rval;
+ }
+ }
+
}