用C#做一個網(wǎng)頁數(shù)據(jù)采集工具
安全中國 www.anqn.com更新時間:2010-06-19 04:24:30
責任編輯:池天
熱 點：
最近做一個網(wǎng)站，該網(wǎng)站需要添加4000多 產(chǎn)品信息，如果用人工方法去別的網(wǎng)站copy那至少要花費半月時間才能完成，所以我個辦法使用c#作出來了一個網(wǎng)頁數(shù)據(jù)采集軟件，通過這個軟件一兩天就完成了幾千產(chǎn)品數(shù)據(jù)的錄入，可見很多工作不是一味用人工去做，作為一個程序員，就是要讓很多讓那些經(jīng)常做重復性的、繁瑣的工作中的人解放出來。下面只是寫了一些核心代碼，而且采集必須要和對應網(wǎng)站相掛鉤，
//提取產(chǎn)品列表頁中產(chǎn)品最終頁的網(wǎng)頁
private void button1_Click(object sender, EventArgs e)
{
if (textBox1.Text.Trim() == "" || textBox2.Text.Trim() == "")
{
MessageBox.Show("網(wǎng)址和域名不能為空!", "信息提示", MessageBoxButtons.OK, MessageBoxIcon.Information);
return;
}
try
{
string Html = inc.GetHtml("http://study.pctoday.net.cn/");
//ArrayList al = inc.GetMatchesStr(Html, "]*?>.*?");
ArrayList al = inc.GetMatchesStr(Html, @"href\s*=\s*(?:[\'\""\s](?<1>[^\""\']*)[\'\""])");//提取鏈接
http://www.egacn.com/" title="Replica Watches:">Replica Watches Buy Full Quality Popular Luxury Watches at Amazing Price, Your One Stop Discount Swiss Watches StoreExclusive Replica Rolex Watches, Tag Heuer Watches Replica, Cartier Watches online Sale!http://www.yuget.nethttp://study.pctoday.net.cn/1_ASP.aspxhttp://study.pctoday.net.cn/
文章來源： http://study.pctoday.net.cn/Article/自己用CSharp做一個網(wǎng)頁數(shù)據(jù)采集工具.html
StringBuilder sb = new StringBuilder();
foreach (object var in al)
{
string a = var.ToString().Replace("\"", "").Replace("'", "");
a = Regex.Replace(a, "href=", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
if (a.StartsWith("/"))
a = textBox2.Text.Trim() + a;
if (!a.StartsWith("http://"))
a = "http://" + a;
sb.Append(a + "\r\n");
}
textBox5.Text = sb.ToString();//把提取到網(wǎng)址輸出到一個textBox，每個鏈接占一行
MessageBox.Show("共提取" + al.Count.ToString() + "個鏈接", "信息提示", MessageBoxButtons.OK, MessageBoxIcon.Information);
}
catch (Exception err)
{
MessageBox.Show("提取出錯!原因：" + err.Message, "信息提示", MessageBoxButtons.OK, MessageBoxIcon.Information);
}
}
//把采集的產(chǎn)品頁面html代碼進行字符串處理，提取需要的代碼，最后保存到本地一個access數(shù)據(jù)庫中，同時提取產(chǎn)品圖片地址并自動現(xiàn)在圖片到本地images文件夾下
private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
{
//填充產(chǎn)品表
Database.ExecuteNonQuery("delete from Tb_Product");
DataTable dt2 = new DataTable();
OleDbConnection conn = new OleDbConnection(Database.ConnectionStrings);
OleDbDataAdapter da = new OleDbDataAdapter("select * from Tb_Product", conn);
OleDbCommandBuilder cb = new OleDbCommandBuilder(da);
da.Fill(dt2);
dt2.Rows.Clear();
BackgroundWorker worker = (BackgroundWorker)sender;//這個是做一個進度條
string[] Urls = textBox5.Text.Trim().ToLower().Replace("\r\n", ",").Split(',');
DataTable dt = new DataTable();
StringBuilder ErrorStr = new StringBuilder();
string html = "", ImageDir = AppDomain.CurrentDomain.BaseDirectory + "Images\\";
//循環(huán)每次采集網(wǎng)址
for (int i = 0; i < Urls.Length; i++)
{
try
{
