You are viewing a plain text version of this content. The canonical link for it is here.
Posted to httpclient-users@hc.apache.org by Thuannd <nh...@yahoo.com> on 2008/11/02 11:15:40 UTC
RE: How to download from URL that requires login?
use vietspider htmlparser build 8,
download
http://downloads.sourceforge.net/binhgiang/HTMLParser2_Build8.zip?use_mirror=
Example: Code login Yahoo Mail.
import java.io.File;
import java.net.URL;
import java.util.List;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.DefaultHttpClient;
import org.vietspider.browser.HttpSessionUtils;
import org.vietspider.common.io.DataWriter;
import org.vietspider.net.client.HttpResponseReader;
import org.vietspider.net.client.WebClient;
/**
* Author : Nhu Dinh Thuan
* nhudinhthuan@yahoo.com
* Nov 2, 2008
*/
public class YahooLogin {
public static void main(String[] args) throws Exception {
WebClient webClient = new WebClient();
String homepage = "http://mail.yahoo.com/";
webClient.setURL(homepage, new URL(homepage));
HttpHost httpHost = webClient.createHttpHost(homepage);
HttpGet httpGet = webClient.createGetMethod(homepage,
"http://www.yahoo.com");
HttpResponse response = webClient.execute(httpHost, httpGet);
HttpEntity entity = response.getEntity();
System.out.println("Login form get: " + response.getStatusLine());
if (entity != null) entity.consumeContent();
System.out.println("Initial set of cookies:");
DefaultHttpClient httpClient = (DefaultHttpClient)
webClient.getHttpClient();
List<Cookie> cookies = httpClient.getCookieStore().getCookies();
if (cookies.isEmpty()) {
System.out.println("None");
} else {
for (int i = 0; i < cookies.size(); i++) {
System.out.println("- " + cookies.get(i).toString());
}
}
HttpSessionUtils httpSession = new HttpSessionUtils(webClient, "ERROR");
StringBuilder builder = new StringBuilder(homepage);
builder.append('\n').append("username:password");
httpSession.login(builder.toString(), "utf-8", new URL(homepage),
homepage);
httpGet = webClient.createGetMethod("http://my.yahoo.com",
"http://www.yahoo.com");
response = webClient.execute(httpHost, httpGet);
entity = response.getEntity();
HttpResponseReader httpResponseReader = new HttpResponseReader();
byte [] bytes = httpResponseReader.readBody(response);
new DataWriter().save(new File("my_yahoo.html"), bytes);
System.out.println("Login form get: " + response.getStatusLine());
if (entity != null) entity.consumeContent();
System.out.println("Post logon cookies:");
cookies = httpClient.getCookieStore().getCookies();
if (cookies.isEmpty()) {
System.out.println("None");
} else {
for (int i = 0; i < cookies.size(); i++) {
System.out.println("- " + cookies.get(i).toString());
}
}
}
}
Code login Gmail:
HttpSessionUtils httpSession = new HttpSessionUtils(webClient, "ERROR");
StringBuilder builder = new
StringBuilder("https://www.google.com/accounts/ServiceLogin?service=mail&passive=true&rm=false&continue=http%3A%2F%2Fmail.google.com%2Fmail%2F%3Fui%3Dhtml%26zy%3Dl&bsv=1k96igf4806cy<mpl=default<mplcache=2");
builder.append('\n').append("username:password");
httpSession.login(builder.toString(), "utf-8", new URL(homepage),
homepage);
Code login to java.net:
String homepage = "http://java.net/";
HttpSessionUtils httpSession = new HttpSessionUtils(webClient,
"ERROR");
StringBuilder builder = new StringBuilder(homepage);
builder.append('\n').append("username:password");
httpSession.login(builder.toString(), "utf-8", new URL(homepage),
homepage);
See more example folder in HTMLParser_Build8
--
View this message in context: http://www.nabble.com/How-to-download-from-URL-that-requires-login--tp20199503p20288466.html
Sent from the HttpClient-User mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: httpclient-users-unsubscribe@hc.apache.org
For additional commands, e-mail: httpclient-users-help@hc.apache.org
RE: How to download from URL that requires login?
Posted by AnuRahim <an...@gmail.com>.
Hi , I have tried this code snippt I am getting the following error
Exception in thread "main" java.lang.RuntimeException: Stub!
at org.apache.http.conn.scheme.SchemeRegistry.<init>(SchemeRegistry.java:4)
at
org.vietspider.net.client.HTTPClientFactory.createRegitry(HTTPClientFactory.java:95)
at
org.vietspider.net.client.HTTPClientFactory.createDefaultHttpClient(HTTPClientFactory.java:82)
at org.vietspider.net.client.WebClient.<init>(WebClient.java:90)
at com.calpine.android.YahooLogin.main(YahooLogin.java:27)
can any one please tel me where I went wrong?
Thuannd wrote:
>
> use vietspider htmlparser build 8,
> download
> http://downloads.sourceforge.net/binhgiang/HTMLParser2_Build8.zip?use_mirror=
>
> Example: Code login Yahoo Mail.
>
> import java.io.File;
> import java.net.URL;
> import java.util.List;
>
> import org.apache.http.HttpEntity;
> import org.apache.http.HttpHost;
> import org.apache.http.HttpResponse;
> import org.apache.http.client.methods.HttpGet;
> import org.apache.http.cookie.Cookie;
> import org.apache.http.impl.client.DefaultHttpClient;
> import org.vietspider.browser.HttpSessionUtils;
> import org.vietspider.common.io.DataWriter;
> import org.vietspider.net.client.HttpResponseReader;
> import org.vietspider.net.client.WebClient;
>
> /**
> * Author : Nhu Dinh Thuan
> * nhudinhthuan@yahoo.com
> * Nov 2, 2008
> */
> public class YahooLogin {
>
> public static void main(String[] args) throws Exception {
> WebClient webClient = new WebClient();
>
> String homepage = "http://mail.yahoo.com/";
> webClient.setURL(homepage, new URL(homepage));
>
> HttpHost httpHost = webClient.createHttpHost(homepage);
> HttpGet httpGet = webClient.createGetMethod(homepage,
> "http://www.yahoo.com");
>
> HttpResponse response = webClient.execute(httpHost, httpGet);
> HttpEntity entity = response.getEntity();
>
> System.out.println("Login form get: " + response.getStatusLine());
> if (entity != null) entity.consumeContent();
>
> System.out.println("Initial set of cookies:");
> DefaultHttpClient httpClient = (DefaultHttpClient)
> webClient.getHttpClient();
> List<Cookie> cookies = httpClient.getCookieStore().getCookies();
> if (cookies.isEmpty()) {
> System.out.println("None");
> } else {
> for (int i = 0; i < cookies.size(); i++) {
> System.out.println("- " + cookies.get(i).toString());
> }
> }
>
> HttpSessionUtils httpSession = new HttpSessionUtils(webClient,
> "ERROR");
>
> StringBuilder builder = new StringBuilder(homepage);
> builder.append('\n').append("username:password");
>
> httpSession.login(builder.toString(), "utf-8", new URL(homepage),
> homepage);
>
> httpGet = webClient.createGetMethod("http://my.yahoo.com",
> "http://www.yahoo.com");
> response = webClient.execute(httpHost, httpGet);
> entity = response.getEntity();
>
>
> HttpResponseReader httpResponseReader = new HttpResponseReader();
> byte [] bytes = httpResponseReader.readBody(response);
> new DataWriter().save(new File("my_yahoo.html"), bytes);
>
> System.out.println("Login form get: " + response.getStatusLine());
> if (entity != null) entity.consumeContent();
>
> System.out.println("Post logon cookies:");
> cookies = httpClient.getCookieStore().getCookies();
> if (cookies.isEmpty()) {
> System.out.println("None");
> } else {
> for (int i = 0; i < cookies.size(); i++) {
> System.out.println("- " + cookies.get(i).toString());
> }
> }
>
> }
> }
>
>
> Code login Gmail:
>
>
> HttpSessionUtils httpSession = new HttpSessionUtils(webClient,
> "ERROR");
>
> StringBuilder builder = new
> StringBuilder("https://www.google.com/accounts/ServiceLogin?service=mail&passive=true&rm=false&continue=http%3A%2F%2Fmail.google.com%2Fmail%2F%3Fui%3Dhtml%26zy%3Dl&bsv=1k96igf4806cy<mpl=default<mplcache=2");
> builder.append('\n').append("username:password");
>
> httpSession.login(builder.toString(), "utf-8", new URL(homepage),
> homepage);
>
> Code login to java.net:
>
> String homepage = "http://java.net/";
> HttpSessionUtils httpSession = new HttpSessionUtils(webClient,
> "ERROR");
> StringBuilder builder = new StringBuilder(homepage);
> builder.append('\n').append("username:password");
> httpSession.login(builder.toString(), "utf-8", new URL(homepage),
> homepage);
>
>
>
> See more example folder in HTMLParser_Build8
>
>
>
>
--
View this message in context: http://www.nabble.com/How-to-download-from-URL-that-requires-login--tp20199503p20361360.html
Sent from the HttpClient-User mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: httpclient-users-unsubscribe@hc.apache.org
For additional commands, e-mail: httpclient-users-help@hc.apache.org