C#/winform采集百度hi文章
public partial class Form1 : Form
{
Thread newth;
public Form1()
{
InitializeComponent();
}
private void buttonGo_Click(object sender, EventArgs e)
{
CheckForIllegalCrossThreadCalls = false; //简单异步线程控制设置
newth = new Thread(new ThreadStart(doit));
newth.Start();
}
void doit()
{ //用HttpWebRequest 对象采集百度hi blog文章
HttpWebRequest webRequest; //请求对象
StreamReader responseReader;//响应对象
string responseData;
html mytml; //自定义html简单处理对象，处理文章页面数据
DataTable dt = new DataTable();//存储文章列表
DataTable dt2 = new DataTable();//存储文章内容
int pagecount = 0;
dt.Columns.Add(new DataColumn("title"));//标题
dt.Columns.Add(new DataColumn("link"));//链接
dt.Columns.Add(new DataColumn("description"));//文章内容
dt.Columns.Add(new DataColumn("pubDate"));//发表时间
dt.Columns.Add(new DataColumn("category"));//文章分类
dt2.Columns.Add(new DataColumn("title"));
dt2.Columns.Add(new DataColumn("link"));
dt2.Columns.Add(new DataColumn("description"));
dt2.Columns.Add(new DataColumn("pubDate"));
dt2.Columns.Add(new DataColumn("category"));
string url = "http://hi.baidu.com/306759613/blog/index/";//文章列表第一页为http://hi.baidu.com/306759613/blog/index/0
string arcurl="http://hi.baidu.com/306759613/blog/item/";//文章所在路径
//find page count
//from index 0
webRequest = WebRequest.Create(url + 0) as HttpWebRequest;
webRequest.Timeout = 3000;//请求延时设置
WebResponse reponse = webRequest.GetResponse();
//以gb2312读取数据
responseReader = new StreamReader(
reponse.GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312")
);
responseData = responseReader.ReadToEnd();//读取整个页面
responseReader.Close();
mytml = new html(responseData);//创建html页面处理对象
List<string> regpsl = mytml.getElementsByRegex(@"/blog/index/[\d]+");//获取分页链接的正则
List<int> pagenum = new List<int>();//存储页码
foreach (string a in regpsl) {
pagenum.Add(int.Parse(a.Replace("/blog/index/", "")));
}
pagecount = pagenum.Max() + 1;//pagenum中最大值为尾页页码，页面从0开始编号，页数为页面数+1
mytml = null;
this.progressBar1.Value = 0; //进度条
for (int i = 0; i < pagecount; i++)
{
webRequest = WebRequest.Create(url+i) as HttpWebRequest;//读取各分页
webRequest.Timeout = 3000;
responseReader = new StreamReader(
webRequest.GetResponse().GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312")
);
responseData = responseReader.ReadToEnd();
responseReader.Close();
mytml = new html(responseData);
DataTable dti = mytml.getAritcleTable();//获取该分页文章列表
this.progressBar1.Value = (i * 100 / pagecount);
this.label1.Text = this.progressBar1.Value + "%"; //进度条
for (int j = 0; j < dti.Rows.Count; j++)
{
dt.Rows.Add(dti.Rows[j].ItemArray);//插入该文章到总文章表
HttpWebRequest subrequest = WebRequest.Create(arcurl+dti.Rows[j][1]+".html") as HttpWebRequest;//读取文章信息
subrequest.Timeout = 3000;
StreamReader subre = new StreamReader (subrequest.GetResponse().GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312"));
string tmphtml = subre.ReadToEnd();
//处理文章页面html开始
int start = tmphtml.IndexOf("<div id=\"blog_text\" class=\"cnt\">")+"<div id=\"blog_text\" class=\"cnt\">".Length;
int end = tmphtml.IndexOf("</div",start);
tmphtml = tmphtml.Substring(start,end-start);//取得文章内容
dt2.Rows.Add(new object[] { dti.Rows[j].ItemArray[0], dti.Rows[j].ItemArray[1], tmphtml, dti.Rows[j].ItemArray[3] });//插入文章数据到文章表
subre.Close();
this.progressBar1.Value = (i * 100 / pagecount) + (j * 25 / dti.Rows.Count);//设置进度条
this.label1.Text = this.progressBar1.Value + "%"; //显示百分比
writeXML(dt2, "f:\\p\\" + dti.Rows[j][1] + ".xml");//将文章以xml格式输出
dt2.Rows.Clear();
subre.Close();
subrequest = null;
}
webRequest = null;
responseReader.Close();
responseReader = null;
responseData = string.Empty;
}
this.progressBar1.Value =this.progressBar1.Maximum;//进度100%
this.label1.Text = this.progressBar1.Value + "%";
this.dataGridView1.DataSource = dt;//显示文章列表数据
writeXML(dt, "f:\\p\\Articel.xml");//输出文章内容数据到xml文件
textBoxDebug.Text = textBoxDebug.Text+ "写入完毕\r\n";
}
/// <summary>
/// 将数据表输出到xml
/// </summary>
/// <param name="dt"></param>
/// <param name="fileName"></param>
public void writeXML(DataTable dt, string fileName)
{
string xmlstr ="<?xml version=\"1.0\" encoding=\"utf-8\"?>\r\n";
xmlstr += "<?xml-stylesheet href=\"t.xsl\" type=\"text/xsl\"?>\r\n";
xmlstr +="<root>\r\n";
dt.TableName = "articels";
System.Xml.XmlDocument xml = new System.Xml.XmlDocument();
for (int k = 0; k < dt.Rows.Count; k++)
{
xmlstr = xmlstr + "<" + dt.TableName + ">\r\n";
for (int l = 0; l < dt.Columns.Count; l++)
{
xmlstr = xmlstr + "<" + dt.Columns[l].ColumnName + ">\r\n<![CDATA[\r\n";
xmlstr = xmlstr + dt.Rows[k][l] + "\r\n";
xmlstr = xmlstr + "]]>\r\n</" + dt.Columns[l].ColumnName + ">\r\n";
}
xmlstr = xmlstr + "</" + dt.TableName + ">\r\n";
}
xmlstr += "</root>\r\n";
StreamWriter w = new StreamWriter(fileName, false, System.Text.Encoding.UTF8);//以utf8保存
w.Write(xmlstr);
w.Close();
}
}
class html
{
string htmltext=string.Empty;
/// <summary>
/// 构造函数
/// </summary>
/// <param name="htmltext"></param>
public html( string htmltext) {
this.htmltext = htmltext;
}
/// <summary>
/// 获取文章列表
/// </summary>
/// <returns></returns>
public DataTable getAritcleTable(){
DataTable dt = new DataTable();
int start = htmltext.IndexOf("div id=\"m_blog\" class=\"modbox\">");//起始位置
int end = htmltext.IndexOf("<div id=\"mod_artclg\" class=\"mod\">");//结束位置
string htm = htmltext.Substring(start-1, end - start -1 );
dt.Columns.Add(new DataColumn("title"));
dt.Columns.Add(new DataColumn("link"));
dt.Columns.Add(new DataColumn("description"));
dt.Columns.Add(new DataColumn("pubDate"));
dt.Columns.Add(new DataColumn("category"));
string title, link, description, pubDate, category,temp;
int nstart, nend;//记录上次提取位置
start = 0;
do
{//遍历html文档 提取文章信息
nstart = htm.IndexOf("<div class=\"tit\">",start) + "<div class=\"tit\">".Length;
if (nstart < start) break;
start = nstart;
nend = htm.IndexOf("</div>",start);
start = nend + 5;
temp = htm.Substring(nstart, nend - nstart );
nstart = temp.IndexOf(">");
nend =temp.IndexOf("</a>");
title = temp.Substring(nstart + 1, nend-nstart-1 );//文章标题
nstart = temp.IndexOf("\"");
nend = temp.IndexOf("\"", nstart + 1);
link = temp.Substring(nstart + 1, nend - nstart-1 );//链接
nstart = link.IndexOf("item/")+"item/".Length;
nend = link.IndexOf(".html");
link = link.Substring(nstart, nend - nstart);//取文件名（去除扩展名）
nstart = htm.IndexOf("<div class=\"date\">", start)+ "<div class=\"date\">".Length;
start = nstart;
nend = htm.IndexOf("</div>", start);
pubDate = htm.Substring(nstart , nend - nstart);//发表日期
start = nend + 5;
nstart = htm.IndexOf("<div class=\"cnt\">", start) + "<div class=\"cnt\">".Length;
start = nstart;
nend = htm.IndexOf("</div>", start);
start = nend + 5;
description = htm.Substring(nstart, nend - nstart );//文章内容
nstart = htm.IndexOf("<div class=\"opt\">", start) + "<div class=\"opt\">".Length;
start = nstart;
nend = htm.IndexOf("</div>", start);
start = nend + 5;
temp = htm.Substring(nstart, nend - nstart );
nstart = temp.IndexOf("：");
nend =temp.IndexOf("</a>");
category=temp.Substring(nstart + 1, nend - nstart - 1); //文章分类
dt.Rows.Add(new string[] { title, link, description, pubDate, category });
} while (nstart > 0);
return dt;
}
}
