網絡爬蟲Java實現抓取網頁內容

package 抓取網頁;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;

public class RetrivePage {

private static HttpClient httpClient = new HttpClient();

public static void main(String[] args) {
//抓取獵兔的首頁，並且輸出出來
try {
RetrivePage.downloadPage("http://www.lietu.com");
} catch (HttpException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}

private static void downloadPage(String path) throws HttpException, IOException {
System.out.println("123123");
InputStream input = null;
OutputStream output = null;
//得到post方法
PostMethod postMethod = new PostMethod(path);
//設置post方法的參數
NameValuePair[] postData = new NameValuePair[2];
postData[0] = new NameValuePair("name","lietu");
postData[1] = new NameValuePair("password","*****");
//把參數添加到請求路徑上去
postMethod.addParameters(postData);
//執行，返回狀態碼
int statusCode = httpClient.executeMethod(postMethod);
System.out.println(statusCode);
if (statusCode == HttpStatus.SC_OK) {
input = postMethod.getResponseBodyAsStream();
//得到文件的名字
String fileName = path.substring(path.lastIndexOf('/')+1);
//獲得文件的輸出流
System.out.println(fileName);
output = new FileOutputStream(fileName);

//輸出到文件中
int tempByte = -1;
while ((tempByte = input.read()) > 0) {
output.write(tempByte);
}
//關閉資源
if (input != null) {
input.close();
}
if (output != null) {
output.close();
}
}
}
}