You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2012/11/21 14:04:15 UTC
svn commit: r1412083 -
/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Author: kwright
Date: Wed Nov 21 13:04:15 2012
New Revision: 1412083
URL: http://svn.apache.org/viewvc?rev=1412083&view=rev
Log:
Hook up interruptible https socket connections
Modified:
manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1412083&r1=1412082&r2=1412083&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java Wed Nov 21 13:04:15 2012
@@ -152,20 +152,19 @@ public class ThrottledFetcher
throws ManifoldCFException
{
// Create the https scheme for this connection
- SSLSocketFactory myFactory;
+ javax.net.ssl.SSLSocketFactory baseFactory;
String trustStoreString;
if (trustStore != null)
{
- myFactory = new SSLSocketFactory(trustStore.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
+ baseFactory = trustStore.getSecureSocketFactory();
trustStoreString = trustStore.getString();
}
else
{
- myFactory = new SSLSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(), new BrowserCompatHostnameVerifier());
+ baseFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
trustStoreString = null;
}
-
- Scheme myHttpsProtocol = new Scheme("https", 443, myFactory);
+
ConnectionBin[] bins = new ConnectionBin[binNames.length];
@@ -405,7 +404,7 @@ public class ThrottledFetcher
// If we have a connection located, activate it.
if (connectionToReuse == null)
- connectionToReuse = new ThrottledConnection(protocol,server,port,authentication,myHttpsProtocol,trustStoreString,bins,
+ connectionToReuse = new ThrottledConnection(protocol,server,port,authentication,baseFactory,trustStoreString,bins,
proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
connectionToReuse.setup(throttleDescription);
return connectionToReuse;
@@ -949,7 +948,7 @@ public class ThrottledFetcher
/** Proxy auth password */
protected final String proxyAuthPassword;
/** Https protocol */
- protected final Scheme myHttpsProtocol;
+ protected final javax.net.ssl.SSLSocketFactory httpsSocketFactory;
/** The thread that is actually doing the work */
protected ExecuteMethodThread methodThread = null;
@@ -960,7 +959,7 @@ public class ThrottledFetcher
/** Constructor. Create a connection with a specific server and port, and
* register it as active against all bins. */
public ThrottledConnection(String protocol, String server, int port, PageCredentials authentication,
- Scheme myHttpsProtocol, String trustStoreString, ConnectionBin[] connectionBins,
+ javax.net.ssl.SSLSocketFactory httpsSocketFactory, String trustStoreString, ConnectionBin[] connectionBins,
String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
{
this.proxyHost = proxyHost;
@@ -972,7 +971,7 @@ public class ThrottledFetcher
this.server = server;
this.port = port;
this.authentication = authentication;
- this.myHttpsProtocol = myHttpsProtocol;
+ this.httpsSocketFactory = httpsSocketFactory;
this.trustStoreString = trustStoreString;
this.connectionBinArray = connectionBins;
this.throttleBinArray = new ThrottleBin[connectionBins.length];
@@ -1252,6 +1251,11 @@ public class ThrottledFetcher
LoginCookies loginCookies)
throws ManifoldCFException, ServiceInterruption
{
+ // Set up scheme
+ SSLSocketFactory myFactory = new SSLSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeoutMilliseconds),
+ new BrowserCompatHostnameVerifier());
+ Scheme myHttpsProtocol = new Scheme("https", 443, myFactory);
+
int resolvedPort;
String displayedPort;
if (port != -1)
@@ -1297,13 +1301,14 @@ public class ThrottledFetcher
if (connManager == null)
{
PoolingClientConnectionManager localConnManager = new PoolingClientConnectionManager();
- // Set up protocol registry
- localConnManager.getSchemeRegistry().register(myHttpsProtocol);
localConnManager.setMaxTotal(1);
localConnManager.setDefaultMaxPerRoute(1);
connManager = localConnManager;
}
-
+
+ // Set up protocol registry
+ connManager.getSchemeRegistry().register(myHttpsProtocol);
+
long startTime = 0L;
if (Logging.connectors.isDebugEnabled())
{
@@ -2155,132 +2160,120 @@ public class ThrottledFetcher
}
}
- /** HTTPClient secure socket factory, which implements SecureProtocolSocketFactory
- protected static class WebSecureSocketFactory implements SecureProtocolSocketFactory
+ /** SSL Socket factory which wraps another socket factory but allows timeout on socket
+ * creation.
+ */
+ protected static class InterruptibleSocketFactory extends javax.net.ssl.SSLSocketFactory
{
- protected javax.net.ssl.SSLSocketFactory socketFactory;
-
- public WebSecureSocketFactory(javax.net.ssl.SSLSocketFactory socketFactory)
+ protected final javax.net.ssl.SSLSocketFactory wrappedFactory;
+ protected final long connectTimeoutMilliseconds;
+
+ public InterruptibleSocketFactory(javax.net.ssl.SSLSocketFactory wrappedFactory, long connectTimeoutMilliseconds)
{
- this.socketFactory = socketFactory;
+ this.wrappedFactory = wrappedFactory;
+ this.connectTimeoutMilliseconds = connectTimeoutMilliseconds;
}
- public Socket createSocket(
- String host,
- int port,
- InetAddress clientHost,
- int clientPort)
- throws IOException, UnknownHostException
- {
- return socketFactory.createSocket(
- host,
- port,
- clientHost,
- clientPort
- );
- }
-
- public Socket createSocket(
- final String host,
- final int port,
- final InetAddress localAddress,
- final int localPort,
- final HttpConnectionParams params
- ) throws IOException, UnknownHostException, ConnectTimeoutException
+ @Override
+ public Socket createSocket()
+ throws IOException
{
- if (params == null)
- {
- throw new IllegalArgumentException("Parameters may not be null");
- }
- int timeout = params.getConnectionTimeout();
- if (timeout == 0)
- {
- return createSocket(host, port, localAddress, localPort);
- }
- else
- {
- // We need to implement a connection timeout somehow - probably with a new thread.
- SocketCreateThread thread = new SocketCreateThread(socketFactory,host,port,localAddress,localPort);
- thread.start();
- try
- {
- // Wait for thread to complete for only a certain amount of time!
- thread.join(timeout);
- // If join() times out, then the thread is going to still be alive.
- if (thread.isAlive())
- {
- // Kill the thread - not that this will necessarily work, but we need to try
- thread.interrupt();
- throw new ConnectTimeoutException("Secure connection timed out");
- }
- // The thread terminated. Throw an error if there is one, otherwise return the result.
- Throwable t = thread.getException();
- if (t != null)
- {
- if (t instanceof java.net.SocketTimeoutException)
- throw (java.net.SocketTimeoutException)t;
- else if (t instanceof org.apache.commons.httpclient.ConnectTimeoutException)
- throw (org.apache.commons.httpclient.ConnectTimeoutException)t;
- else if (t instanceof InterruptedIOException)
- throw (InterruptedIOException)t;
- else if (t instanceof IOException)
- throw (IOException)t;
- else if (t instanceof UnknownHostException)
- throw (UnknownHostException)t;
- else if (t instanceof Error)
- throw (Error)t;
- else if (t instanceof RuntimeException)
- throw (RuntimeException)t;
- throw new Error("Received an unexpected exception: "+t.getMessage(),t);
- }
- return thread.getResult();
- }
- catch (InterruptedException e)
- {
- throw new InterruptedIOException("Interrupted: "+e.getMessage());
- }
- }
+ // Socket isn't open
+ return wrappedFactory.createSocket();
}
-
+
+ @Override
public Socket createSocket(String host, int port)
throws IOException, UnknownHostException
{
- return socketFactory.createSocket(
- host,
- port
- );
+ return fireOffThread(InetAddress.getByName(host),port,null,-1);
}
- public Socket createSocket(
- Socket socket,
- String host,
- int port,
- boolean autoClose)
+ @Override
+ public Socket createSocket(InetAddress host, int port)
+ throws IOException
+ {
+ return fireOffThread(host,port,null,-1);
+ }
+
+ @Override
+ public Socket createSocket(String host, int port, InetAddress localHost, int localPort)
throws IOException, UnknownHostException
{
- return socketFactory.createSocket(
- socket,
- host,
- port,
- autoClose
- );
+ return fireOffThread(InetAddress.getByName(host),port,localHost,localPort);
}
-
- public boolean equals(Object obj)
+
+ @Override
+ public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort)
+ throws IOException
{
- if (obj == null || !(obj instanceof WebSecureSocketFactory))
- return false;
- // Each object is unique
- return super.equals(obj);
+ return fireOffThread(address,port,localAddress,localPort);
}
-
- public int hashCode()
+
+ @Override
+ public Socket createSocket(Socket s, String host, int port, boolean autoClose)
+ throws IOException
+ {
+ // Socket's already open
+ return wrappedFactory.createSocket(s,host,port,autoClose);
+ }
+
+ @Override
+ public String[] getDefaultCipherSuites()
{
- return super.hashCode();
+ return wrappedFactory.getDefaultCipherSuites();
}
+
+ @Override
+ public String[] getSupportedCipherSuites()
+ {
+ return wrappedFactory.getSupportedCipherSuites();
+ }
+
+ protected Socket fireOffThread(InetAddress address, int port, InetAddress localHost, int localPort)
+ throws IOException
+ {
+ SocketCreateThread thread = new SocketCreateThread(wrappedFactory,address,port,localHost,localPort);
+ thread.start();
+ try
+ {
+ // Wait for thread to complete for only a certain amount of time!
+ thread.join(connectTimeoutMilliseconds);
+ // If join() times out, then the thread is going to still be alive.
+ if (thread.isAlive())
+ {
+ // Kill the thread - not that this will necessarily work, but we need to try
+ thread.interrupt();
+ throw new ConnectTimeoutException("Secure connection timed out");
+ }
+ // The thread terminated. Throw an error if there is one, otherwise return the result.
+ Throwable t = thread.getException();
+ if (t != null)
+ {
+ if (t instanceof java.net.SocketTimeoutException)
+ throw (java.net.SocketTimeoutException)t;
+ else if (t instanceof ConnectTimeoutException)
+ throw (ConnectTimeoutException)t;
+ else if (t instanceof InterruptedIOException)
+ throw (InterruptedIOException)t;
+ else if (t instanceof IOException)
+ throw (IOException)t;
+ else if (t instanceof Error)
+ throw (Error)t;
+ else if (t instanceof RuntimeException)
+ throw (RuntimeException)t;
+ throw new Error("Received an unexpected exception: "+t.getMessage(),t);
+ }
+ return thread.getResult();
+ }
+ catch (InterruptedException e)
+ {
+ throw new InterruptedIOException("Interrupted: "+e.getMessage());
+ }
+ }
+
}
- */
/** Create a secure socket in a thread, so that we can "give up" after a while if the socket fails to connect.
*/
@@ -2288,7 +2281,7 @@ public class ThrottledFetcher
{
// Socket factory
protected javax.net.ssl.SSLSocketFactory socketFactory;
- protected String host;
+ protected InetAddress host;
protected int port;
protected InetAddress clientHost;
protected int clientPort;
@@ -2300,7 +2293,7 @@ public class ThrottledFetcher
/** Create the thread */
public SocketCreateThread(javax.net.ssl.SSLSocketFactory socketFactory,
- String host,
+ InetAddress host,
int port,
InetAddress clientHost,
int clientPort)
@@ -2317,7 +2310,10 @@ public class ThrottledFetcher
{
try
{
- rval = socketFactory.createSocket(host,port,clientHost,clientPort);
+ if (clientHost == null)
+ rval = socketFactory.createSocket(host,port);
+ else
+ rval = socketFactory.createSocket(host,port,clientHost,clientPort);
}
catch (Throwable e)
{