网络爬虫(抓取)正则表达式 (多线程协作)
1.多线程调用界面后台代码 using System; using System.Collections.Generic; using System.Linq; using System.Web; using System.Web.UI; using System.Web.UI.WebControls; using HraWeb.Common; using WebApp.Common; using Contract.Domain; using System.Collections; using System.IO; using System.Net; using System.Runtime.Serialization.Json; using System.Text; using System.Text.RegularExpressions; using System.Threading; using Common; using Elmah; using Framework; using ThreadTemplate; using System.Diagnostics; namespace HraWeb { public partial class CcrCompanyManage : JQEntityManage<Contract.Domain.CcrCompanyFundamental> { private Spring.Caching.ICache cache; private Spring.Caching.ICache SpringCache { get { if (cache == null) cache = (Spring.Caching.ICache)ctx.GetObject("AspNetCache"); return cache; } set { cache = value; } } public static Dictionary<int, IList<CcrCompanyFundamental>> dic = new Dictionary<int, IList<CcrCompanyFundamental>>(); private void ConstractionPrePageIndexData(int pageIndex, int pageSize, IList<CcrCompanyFundamental> funmentals) { int start = (pageIndex - 1) * pageSize + 1; int end = start + pageSize - 1; for (int i = start; i <= end; i++) { if (!dic.ContainsKey(pageIndex)) { dic.Add(pageIndex, new List<CcrCompanyFundamental>() { }); } dic[pageIndex].Add(funmentals[i - 1]); } } private ArrayList GetRatingInfo() { //Dao2 = GetDao(); IList<CcrCreditScoreInfo> temps = Holworth.Utility.HraUtility.ListToT<CcrCreditScoreInfo>(Dao.FindList(new QueryInfo("CcrCreditScoreInfo"))); List<int?> fids = temps.Select(x => x.FundamentalId).ToList(); ArrayList ratingClassPaimingStates = new ArrayList(); QueryInfo finfo = new QueryInfo("CcrCompanyFundamental"); WebClient wc = new WebClient(); IList<CcrCompanyFundamental> funmentals = Holworth.Utility.HraUtility.ListToT<CcrCompanyFundamental>(Dao.FindList(finfo)); funmentals = (from f in funmentals where !fids.Contains(int.Parse(f.Id)) select f).ToList(); int pageIndex = 1; int pageSize = 1; int totalCount = funmentals.Count;//总记录数,亦总开线程数 int totalPage = (totalCount - 1) / pageSize + 1;// //处理前面的n-1页的数据 for (pageIndex = 1; pageIndex < totalPage; pageIndex++) { ConstractionPrePageIndexData(pageIndex, pageSize, funmentals); } //处理最后一页的数据 int LastFirstOne = (pageIndex - 1) * pageSize + 1; for (int j = LastFirstOne; j <= totalCount; j++) { if (!dic.ContainsKey(pageIndex)) { dic.Add(pageIndex, new List<CcrCompanyFundamental>()); } dic[pageIndex].Add(funmentals[j - 1]); } //多线程的最大并发数 int maxPoolThread = 100; int totalThreadNum = dic.Count; //当前正在运行的线程 var runingHt = new Dictionary<int, clsSubThread>(); //处于等待队列的未运行的线程 var unRunHt = new Dictionary<int, clsSubThread>(); //选取maxPoolThread个线程加入运行队列,其余放入未运行的等待队列 for (int i = 1; i <= totalThreadNum; i++) { clsSubThread th = new clsSubThread(i, dic[i]); if (i <= maxPoolThread) { runingHt.Add(i,th); th.Start(); } else { unRunHt.Add(i, th); } } while (true) { //初始化完成队列,用于存取已经执行完的线程的id var stepFinishList = new List<int>(); //将完成的线程放入完成队列 foreach (int tid in runingHt.Keys) { var t = runingHt[tid]; if (t.IsStopped) { stepFinishList.Add(tid); } } //1.遍历完成队列,从当前运行的线程队列中移除该线程 //2.对完成的线程执行回调,将数据持久化到数据库 //3.如果等待队列中还有数据,获取等待队列中的第一个,并执行该线程,将该线程从等待队列移除,加入到运行队列 foreach (int tid in stepFinishList) { Thread t1 = new Thread(new ParameterizedThreadStart(SaveOrUpdate)); t1.Start(runingHt[tid].ReturnList); runingHt.Remove(tid); if (unRunHt.Count > 0) { clsSubThread unRunThread = unRunHt.First().Value; var unRunTid= unRunHt.First().Key; unRunThread.Start(); runingHt.Add(unRunTid, unRunThread); unRunHt.Remove(unRunTid); } } //所有线程都完成后,跳出循环 if (runingHt.Count == 0 && unRunHt.Count == 0) { break; } } return ratingClassPaimingStates; } ///线程完成之后的回调动作,将返回的List保存到数据库 private void SaveOrUpdate(object o) { IList list = (IList)o; Dao.SaveOrUpdateAll(list); } protected override void Page_Load(object sender, EventArgs e) { base.Page_Load(sender, e); } ///click事件抓取网页信息,通过多线程协作插入到数据表 protected void Button1_Click(object sender, EventArgs e) { GetRatingInfo(); } } } 2.封装的线程类 using System.Collections.Generic; using System.Net; using System.Text; using System.Text.RegularExpressions; using Contract.Domain; using System.Web; namespace ThreadTemplate { using System; using System.Threading; using System.IO; /// <summary> /// Summary description for clsSubThread. /// </summary> public class clsSubThread : IDisposable { private Thread thdSubThread = null; private Mutex mUnique = new Mutex(); private bool blnIsStopped; private bool blnSuspended; private bool blnStarted; private IList<CcrCompanyFundamental> clist; private int threadId; public int ThreadId { get { return threadId; } set { threadId = value; } } private IList<CcrCreditScoreInfo> paiming; public bool IsStopped { get { return blnIsStopped; } } public bool IsSuspended { get { return blnSuspended; } } public IList<CcrCreditScoreInfo> ReturnList { get { return paiming; } } public IList<CcrCompanyFundamental> CList { set { clist = value; } } public clsSubThread(int key,IList<CcrCompanyFundamental> pclist) { threadId = key; paiming=new List<CcrCreditScoreInfo>(); // // TODO: Add constructor logic here // blnIsStopped = true; blnSuspended = false; blnStarted = false; clist = pclist; } /// <summary> /// Start sub-thread /// </summary> public void Start() { if (!blnStarted) { thdSubThread = new Thread(new ThreadStart(SubThread)); blnIsStopped = false; blnStarted = true; thdSubThread.Start(); } } /// <summary> /// Thread entry function 线程执行方法,从网站中用正则表达式,抓取需要的数据 /// </summary> private void SubThread() { paiming = new List<CcrCreditScoreInfo>(); WebClient wc=new WebClient(); // do { for (int i = 0; i < clist.Count; i++) { CcrCompanyFundamental company = clist[i]; CcrCreditScoreInfo c = new CcrCreditScoreInfo(); c.FundamentalId = int.Parse(company.Id); c.CompanyName = company.CompanyName; string keyword = "湖南艾华集团股份有限公司"; using (Stream stream = wc.OpenRead("http://bgcheck.cn/MemberCenter/FirmCredit/Search.html?Keywords=" + keyword)) { using (StreamReader sr = new StreamReader(stream, Encoding.UTF8)) { string content = sr.ReadToEnd(); string ratingClasspatern = @"(?<=\[信用等级:([\s\S]*?)<a(.*)?[^>]*?>)([\s\S]*?)(?=</a>)"; string ratingSequencepatern = @"(?<=信用排名:([\s\S]*?)<span(.*)?[^>]*?>)([\s\S]*?)(?=</span>)"; string ratingStatepatern = @"(?<=信用状况:([\s\S]*?)<span(.*)?[^>]*?>)([\s\S]*?)(?=</span>)"; MatchCollection ratingClassmatches = Regex.Matches(content, ratingClasspatern); MatchCollection ratingSequencematches = Regex.Matches(content, ratingSequencepatern); MatchCollection ratingStatematches = Regex.Matches(content, ratingStatepatern); string ratingClass = string.Empty; string ratingSequence = string.Empty; string ratingState = string.Empty; foreach (Match match in ratingClassmatches) { ratingClass = match.Groups[0].Value; break; } foreach (Match match in ratingSequencematches) { ratingSequence = match.Groups[0].Value; break; } foreach (Match match in ratingStatematches) { ratingState = match.Groups[0].Value; break; } c.RatingClass = ratingClass; c.RatingSequence = ratingSequence; c.RatingState = ratingState; paiming.Add(c); } } } // Release CPU here } this.Stop(); //while (blnIsStopped == false); } /// <summary> /// Suspend sub-thread /// </summary> public void Suspend() { if (blnStarted && !blnSuspended) { blnSuspended = true; mUnique.WaitOne(); } } /// <summary> /// Resume sub-thread /// </summary> public void Resume() { if (blnStarted && blnSuspended) { blnSuspended = false; mUnique.ReleaseMutex(); } } /// <summary> /// Stop sub-thread /// </summary> public void Stop() { if (blnStarted) { if (blnSuspended) Resume(); blnStarted = false; blnIsStopped = true; thdSubThread.Join(); } } #region IDisposable Members /// <summary> /// Class resources dispose here /// </summary> public void Dispose() { // TODO: Add clsSubThread.Dispose implementation Stop();//Stop thread first GC.SuppressFinalize(this); } #endregion } }