流程
1:提供要抓取的網(wǎng)頁地址（列表）
2:提取網(wǎng)頁列表中目標(biāo)所有LINK
3:抓取LINK中的所有網(wǎng)頁(爬蟲)
4:解析正文內(nèi)容
5:存入數(shù)據(jù)庫
一、抓取任務(wù)（主程序）
package com.test;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
public class CatchJob {
public String catchJob(String url){
String document= null;
List allLinks = null;
try {
// 獲取網(wǎng)頁內(nèi)容
document = ExtractPage.getContentByUrl(url);
// 獲取頁面指定內(nèi)容的Link
allLinks = ExtractPage.getLinksByConditions(document, "http://www.free9.net/others/gift/");
if(allLinks!=null&&!allLinks.isEmpty()){
for(int i=0;i<allLinks.size();i++){
String link = (String)allLinks.get(i);
String content = ExtractPage.getContentByUrl(link);
ExtractPage.readByHtml(content);
}
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return "success";
}
public static void main(String[] args){
Long startTime = System.currentTimeMillis();
System.out.println(">>start.......");
String httpProxyHost = "211.167.0.131";//default http proxy
String httpProxyPort = "80"; //default http port
System.getProperties().setProperty( "http.proxyHost", httpProxyHost);
System.getProperties().setProperty( "http.proxyPort", httpProxyPort);
CatchJob job = new CatchJob();
//System.out.println(job.catchJob("http://www.free9.net/others/gift/2008-12-09/12288046534312.htm"));
System.out.println(job.catchJob("http://www.free9.net/others/gift/"));
Date date = new Date(System.currentTimeMillis()-startTime);
SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss ");
String s = sdf.format(date);
System.out.println(">>end.......USE"+s+"秒");
}
}
二、抓取網(wǎng)頁內(nèi)容，并解析
package com.test;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.TitleTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
public class ExtractPage {
//抓取頁面內(nèi)容
public static String getContentByUrl(String url){
System.out.println("**********抓取頁面內(nèi)容***********");
StringBuffer document= null;
URL targetUrl;
try {
targetUrl = new URL(url);
HttpURLConnection con = (HttpURLConnection) targetUrl.openConnection();
con.setFollowRedirects(true);
con.setInstanceFollowRedirects(false);
con.connect();
BufferedReader br = new BufferedReader(new InputStreamReader(con.getInputStream(),"gb2312"));
String s = "";
document= new StringBuffer();
while ((s = br.readLine()) != null) {
document.append(s+"\r\n");
}
s=null;
br.close();
return document.toString();
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
// 按頁面方式處理.解析標(biāo)準(zhǔn)的html頁面
public static void readByHtml(String result) throws Exception
{
System.out.println("**********按頁面方式處理.解析標(biāo)準(zhǔn)的html頁面***********");
Parser parser;
NodeList nodelist;
parser = Parser.createParser(result, "utf8");
NodeFilter textFilter = new NodeClassFilter(Div.class);
//NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);
NodeFilter titleFilter = new NodeClassFilter(TitleTag.class);
OrFilter lastFilter = new OrFilter();
lastFilter.setPredicates(new NodeFilter[] { textFilter,titleFilter});
nodelist = parser.parse(lastFilter);
Node[] nodes = nodelist.toNodeArray();
StringBuffer page = new StringBuffer();
String id = "";
for (int i = 0; i < nodes.length; i++) {
Node node = nodes[i];
if (node instanceof Div) {
Div textnode = (Div) node;
id = textnode.getAttribute("id");
if ("Zoom".equals(id)) {
//System.out.println(textnode.getChild(5).toHtml());
page.append(textnode.getChild(5).toHtml().toString());
page.append(textnode.getChild(6).toHtml().toString());
}
}else if (node instanceof TitleTag) {
TitleTag titlenode = (TitleTag) node;
page.append(titlenode.getTitle().substring(0, titlenode.getTitle().indexOf("|")));
}
/*
else if (node instanceof LinkTag) {
LinkTag link = (LinkTag) node;
line = link.getLink();
} else if (node instanceof TitleTag) {
TitleTag titlenode = (TitleTag) node;
line = titlenode.getTitle();
}
*/
//if (isTrimEmpty(line))
//continue;
}
System.out.println(page.toString());
}
//獲取頁面指定內(nèi)容的Link
public static List getLinksByConditions(String result,String coditions){
System.out.println("**********//獲取頁面指定內(nèi)容的Link***********");
List links = null;
Parser parser;
NodeList nodelist;
parser = Parser.createParser(result, "utf8");
NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);
try {
links = new ArrayList();
nodelist = parser.parse(linkFilter);
Node[] nodes = nodelist.toNodeArray();
for (int i = 0; i < nodes.length; i++) {
Node node = nodes[i];
if (node instanceof LinkTag) {
LinkTag link = (LinkTag) node;
if(link.toHtml().indexOf(coditions)!=-1&&link.toHtml().indexOf("index")==-1&&link.toHtml().indexOf(".htm")!=-1){
System.out.println(link.toHtml());
links.add(link.getLink());
}
}
//if (isTrimEmpty(line))
//continue;
}
} catch (ParserException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return links;
}
}
