You are viewing a plain text version of this content. The canonical link for it is here.
Posted to taglibs-dev@jakarta.apache.org by ca...@apache.org on 2003/02/19 05:52:06 UTC
cvs commit: jakarta-taglibs/scrape/src/org/apache/taglibs/scrape HttpConnection.java PageTag.java PageData.java
catlett 2003/02/18 20:52:06
Modified: scrape/src/org/apache/taglibs/scrape HttpConnection.java
PageTag.java PageData.java
Log:
changed the way data is passed, it is passed via the PageData object instead of seperate variables
Revision Changes Path
1.7 +46 -23 jakarta-taglibs/scrape/src/org/apache/taglibs/scrape/HttpConnection.java
Index: HttpConnection.java
===================================================================
RCS file: /home/cvs/jakarta-taglibs/scrape/src/org/apache/taglibs/scrape/HttpConnection.java,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- HttpConnection.java 3 May 2002 16:14:24 -0000 1.6
+++ HttpConnection.java 19 Feb 2003 04:52:05 -0000 1.7
@@ -128,7 +128,7 @@
* boolean value determines if the connection to to travel via a secure
* connection
*/
- private boolean ssl = false;
+ //private boolean ssl = false;
/**
* base64 encoded username and password used for proxy authentication
*/
@@ -169,6 +169,10 @@
* pageContext of the servlet used for logging
*/
private PageContext pageContext = null;
+ /**
+ * pagedata object that represents the page to be scraped
+ */
+ private PageData pagedata = null;
/**
* constructor creates an instance of HttpConnection and calls the super class
@@ -177,12 +181,16 @@
* @param url - url of the http server to connect to
*
*/
- public HttpConnection(URL url, PageContext pc) {
+ public HttpConnection(URL url, PageData pd, PageContext pc) {
//public HttpConnection(URL url, boolean secure, String pass, PageContext pc) {
this.url = url;
//ssl = secure;
//sslpass = pass;
pageContext = pc;
+ pagedata = pd;
+ if (pagedata.getProxyServer() != null) {
+ useproxy = true;
+ }
}
/**
@@ -228,20 +236,20 @@
// get port number
if ((port = getURL().getPort()) == -1) {
- if (ssl)
+ if (pagedata.getSSL())
port = 443; // default port for https
else
port = 80; // default port for http
}
// create the socket for this connection
- //if (!ssl) {
+ //if (!pagedata.getSSL()) {
// make a connection for http
if (useproxy) {
// create socket to the proxy server
- InetAddress proxy = InetAddress.getByName(pserver);
+ InetAddress proxy = InetAddress.getByName(pagedata.getProxyServer());
// make the connection to the proxy server
- socket = new Socket(proxy, pport);
+ socket = new Socket(proxy, pagedata.getProxyPort());
} else {
// create the Internet address object for this url
InetAddress dst = InetAddress.getByName(getURL().getHost());
@@ -365,19 +373,34 @@
*
*/
public void sendRequest() throws IOException {
- System.out.println("sendRequest() useproxy = " + useproxy + "\n" +
- getRequestMethod() + " " + url.toString() + " HTTP/1.1\r\n");
+ ArrayList headers = pagedata.getHeaders();
+ //System.out.println("sendRequest() useproxy = " + useproxy + "\n" +
+ // getRequestMethod() + " " + url.toString() + " HTTP/1.1\r\n");
// send request to the server via http check for not ssl because if a proxy
// and ssl are in use the request is to be tunneled through the proxy
// connection and is already setup
- if (useproxy && !ssl) {
+ if (useproxy && !pagedata.getSSL()) {
send(out, getRequestMethod() + " " + url.toString() + " HTTP/1.1\r\n");
- if (auth != null)
- send(out, "Proxy-Authorization: " + auth + "\r\n");
+ if (pagedata.getAuth() != null)
+ send(out, "Proxy-Authorization: " + pagedata.getAuth() + "\r\n");
+ if (headers != null) {
+ ArrayList name = (ArrayList) headers.get(0);
+ ArrayList value = (ArrayList) headers.get(1);
+ for (int i = 0; i < name.size(); i++) {
+ send(out, name.get(i) + ": " + value.get(i) + "\r\n");
+ }
+ }
send(out, "Connection: close\r\n");
} else {
send(out, getRequestMethod() + " " + url.getFile() + " HTTP/1.1\r\n");
send(out, "Host: " + url.getHost() + "\r\n");
+ if (headers != null) {
+ ArrayList name = (ArrayList) headers.get(0);
+ ArrayList value = (ArrayList) headers.get(1);
+ for (int i = 0; i < name.size(); i++) {
+ send(out, name.get(i) + ": " + value.get(i) + "\r\n");
+ }
+ }
}
//send(sslout, "Accept: text/plain, text/html \r\n");
send(out, "\r\n");
@@ -417,21 +440,21 @@
*
*/
boolean makeTunnelConnection() throws IOException {
- System.out.println("makeTunnelConnection() sending CONNECT to proxy CONNECT "
- + url.getHost() + ":" + port + " HTTP/1.1");
+ //System.out.println("makeTunnelConnection() sending CONNECT to proxy CONNECT "
+ // + url.getHost() + ":" + port + " HTTP/1.1");
// create socket to the proxy server
- InetAddress proxy = InetAddress.getByName(pserver);
+ InetAddress proxy = InetAddress.getByName(pagedata.getProxyServer());
// make the connection to the proxy server
- socket = new Socket(proxy, pport);
+ socket = new Socket(proxy, pagedata.getProxyPort());
// get the outputstream
out = socket.getOutputStream();
send(out, "CONNECT " + url.getHost() + ":" + port + " HTTP/1.0\r\n");
- if (auth != null) {
- //System.out.println("sending the auth line Proxy-Authorization: " + auth);
- send(out, "Proxy-Authorization: " + auth + "\r\n");
+ if (pagedata.getAuth() != null) {
+ //System.out.println("sending the auth line Proxy-Authorization: " + auth);
+ send(out, "Proxy-Authorization: " + pagedata.getAuth() + "\r\n");
}
send(out, "\r\n");
@@ -480,7 +503,7 @@
KeyStore ks = KeyStore.getInstance("JKS");
ks.load((InputStream)filein, null);
System.out.println("password = " + sslpass);
- char[] password = sslpass.toCharArray();
+ char[] password = pagedata.getSslPass.toCharArray();
kmf.init(ks, password);
context.init(kmf.getKeyManagers(), null, null);
sslfactory = context.getSocketFactory();
@@ -740,7 +763,7 @@
}
c = in.read();
}
- System.out.println("The results of the read = " + result);
+ //System.out.println("The results of the read = " + result);
return result;
}
}
1.6 +13 -3 jakarta-taglibs/scrape/src/org/apache/taglibs/scrape/PageTag.java
Index: PageTag.java
===================================================================
RCS file: /home/cvs/jakarta-taglibs/scrape/src/org/apache/taglibs/scrape/PageTag.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- PageTag.java 30 Apr 2002 21:53:18 -0000 1.5
+++ PageTag.java 19 Feb 2003 04:52:06 -0000 1.6
@@ -308,6 +308,16 @@
}
/**
+ * set the name and value of any extra headers to be sent
+ *
+ * @param name string that is the name of an extra header to be sent
+ * @param value string that is the value of an extra header to be sent
+ */
+ public final void setHeader(String name, String value) {
+ pagedata.setHeader(name, value);
+ }
+
+ /**
* method sets the scrapedata object in the hashmap scrapes in the
* application scope pagedata object
*
1.13 +96 -12 jakarta-taglibs/scrape/src/org/apache/taglibs/scrape/PageData.java
Index: PageData.java
===================================================================
RCS file: /home/cvs/jakarta-taglibs/scrape/src/org/apache/taglibs/scrape/PageData.java,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -r1.12 -r1.13
--- PageData.java 7 Sep 2002 18:25:11 -0000 1.12
+++ PageData.java 19 Feb 2003 04:52:06 -0000 1.13
@@ -65,6 +65,7 @@
import java.io.*;
import java.net.*;
import javax.servlet.jsp.*;
+import sun.misc.BASE64Encoder;
import org.apache.oro.text.regex.*;
/**
@@ -149,6 +150,14 @@
* password to the client keystore for ssl client side authentication
*/
private String sslclientpass = null;
+ /**
+ * list of names extra headers to add
+ */
+ private ArrayList name = new ArrayList(10);
+ /**
+ * list of values of extra headers to add
+ */
+ private ArrayList value = new ArrayList(10);
/**
* constructor for the class simply creates an instance of the PageData
@@ -189,7 +198,7 @@
pagedata.setAuth(name, pass);
/*
// set secure for the connection
- pagedata.setSecure(ssl);
+ pagedata.setSSL(ssl);
// set the keystore password for client side authentication
pagedata.setClientPass(sslpass);
// if secure == true set system property so that https will be
@@ -295,6 +304,39 @@
}
/**
+ * set the name and value of any extra headers to be sent
+ *
+ * @param name string that is the name of an extra header to be sent
+ * @param value string that is the value of an extra header to be sent
+ */
+ protected final void setHeader(String name, String value) {
+ if (name == null) {
+ this.name = new ArrayList(5);
+ this.value = new ArrayList(5);
+ }
+ this.name.add(name);
+ this.value.add(value);
+ }
+
+ /**
+ * get the http headers
+ *
+ * @return name ArrayList of the names of http headers to be sent
+ * @return value ArrayList or the values of http headers to be sent
+ *
+ */
+ public ArrayList getHeaders() {
+ if (name == null) {
+ return null;
+ } else {
+ ArrayList list = new ArrayList(2);
+ list.add(name);
+ list.add(value);
+ return list;
+ }
+ }
+
+ /**
* setter method for newflag only called to set newflag to false
*
*/
@@ -323,6 +365,16 @@
}
/**
+ * get the value of the proxy port
+ *
+ * @return - int the proxy port number
+ *
+ */
+ public final int getProxyPort() {
+ return pport;
+ }
+
+ /**
* set the value of proxy server
*
* @param value the proxy server to use for the connection
@@ -332,6 +384,15 @@
pserver = value;
}
+ /**
+ * get the value of the proxy port
+ *
+ * @return - String the name or the proxy server
+ *
+ */
+ public final String getProxyServer() {
+ return pserver;
+ }
/**
* set the pass word to access the client keystore
*
@@ -352,7 +413,18 @@
*/
public final void setAuth(String name, String pass) {
if (name != null && pass != null)
- auth = name + ":" + pass;
+ auth = "Basic " +
+ new BASE64Encoder().encode((name + ":" + pass).getBytes());
+ }
+
+ /**
+ * get the base64 encoded auth string for proxy authorization
+ *
+ * @return - String base64 encoded authorization for the proxy server
+ *
+ */
+ public final String getAuth() {
+ return auth;
}
/**
@@ -362,11 +434,20 @@
* is false
*
*/
- public final void setSecure(boolean value) {
+ public final void setSSL(boolean value) {
ssl = value;
}
/**
+ * get secure
+ *
+ * @return - boolean value of secure flag
+ */
+ public final boolean getSSL() {
+ return ssl;
+ }
+
+ /**
* getter method for the key set of the HashMap scrapes
*
* @return the keys that the scrapes for this page are hashed to
@@ -492,12 +573,14 @@
// create thread page if it doesn't exist check for a
// proxy connection
try {
- if (pport != -1 && pserver != null)
+ page = new Page(url, this, pc);
+ /*if (pport != -1 && pserver != null)
page = new Page(url, this, pc, pport, pserver, auth);
//page = new Page(url, this, pc, pport, pserver, ssl, auth);
else
page = new Page(url, this, pc);
//page = new Page(url, this, pc, ssl);
+ */
} catch (MalformedURLException mue) {
pc.getServletContext().log("PageData.scrapePage(): "
+ mue.getMessage());
@@ -541,9 +624,9 @@
}
/**
- * Create an http request for the specified URL, check to see if time has
- * elapsed, if so get page, check last modified header of page, and if
- * necessary make the request
+ * Create an http request for the specified URL, check to see if time has
+ * elapsed, if so get page, check last modified header of page, and if
+ * necessary make the request
*
*/
class Page extends Thread {
@@ -633,13 +716,14 @@
// make http connection to url
try {
// create new HttpUrlConnection
- if (!proxy)
+ connection = new HttpConnection(url, pagedata, pageContext);
+ /*if (!proxy)
//connection = new HttpConnection(url, ssl, pageContext);
connection = new HttpConnection(url, pageContext);
else
//new HttpConnection(url, pport, pserver, ssl, authstring);
connection = new HttpConnection(url, pport, pserver,
- authstring, pageContext);
+ authstring, pageContext);*/
connection.setRequestMethod("HEAD");
connection.connect();
connection.sendRequest();
---------------------------------------------------------------------
To unsubscribe, e-mail: taglibs-dev-unsubscribe@jakarta.apache.org
For additional commands, e-mail: taglibs-dev-help@jakarta.apache.org