1.多线程调用界面后台代码
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using HraWeb.Common;
using WebApp.Common;
using Contract.Domain;
using System.Collections;
using System.IO;
using System.Net;
using System.Runtime.Serialization.Json;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using Common;
using Elmah;
using Framework;
using ThreadTemplate;
using System.Diagnostics;
namespace HraWeb
{
public partial class CcrCompanyManage : JQEntityManage<Contract.Domain.CcrCompanyFundamental>
{
private Spring.Caching.ICache cache;
private Spring.Caching.ICache SpringCache
{
get
{
if (cache == null)
cache = (Spring.Caching.ICache)ctx.GetObject("AspNetCache");
return cache;
}
set { cache = value; }
}
public static Dictionary<int, IList<CcrCompanyFundamental>> dic = new Dictionary<int, IList<CcrCompanyFundamental>>();
private void ConstractionPrePageIndexData(int pageIndex, int pageSize, IList<CcrCompanyFundamental> funmentals)
{
int start = (pageIndex - 1) * pageSize + 1;
int end = start + pageSize - 1;
for (int i = start; i <= end; i++)
{
if (!dic.ContainsKey(pageIndex))
{
dic.Add(pageIndex, new List<CcrCompanyFundamental>() { });
}
dic[pageIndex].Add(funmentals[i - 1]);
}
}
private ArrayList GetRatingInfo()
{
//Dao2 = GetDao();
IList<CcrCreditScoreInfo> temps = Holworth.Utility.HraUtility.ListToT<CcrCreditScoreInfo>(Dao.FindList(new QueryInfo("CcrCreditScoreInfo")));
List<int?> fids = temps.Select(x => x.FundamentalId).ToList();
ArrayList ratingClassPaimingStates = new ArrayList();
QueryInfo finfo = new QueryInfo("CcrCompanyFundamental");
WebClient wc = new WebClient();
IList<CcrCompanyFundamental> funmentals = Holworth.Utility.HraUtility.ListToT<CcrCompanyFundamental>(Dao.FindList(finfo));
funmentals = (from f in funmentals where !fids.Contains(int.Parse(f.Id)) select f).ToList();
int pageIndex = 1;
int pageSize = 1;
int totalCount = funmentals.Count;//总记录数，亦总开线程数
int totalPage = (totalCount - 1) / pageSize + 1;//
//处理前面的n-1页的数据
for (pageIndex = 1; pageIndex < totalPage; pageIndex++)
{
ConstractionPrePageIndexData(pageIndex, pageSize, funmentals);
}
//处理最后一页的数据
int LastFirstOne = (pageIndex - 1) * pageSize + 1;
for (int j = LastFirstOne; j <= totalCount; j++)
{
if (!dic.ContainsKey(pageIndex))
{
dic.Add(pageIndex, new List<CcrCompanyFundamental>());
}
dic[pageIndex].Add(funmentals[j - 1]);
}
//多线程的最大并发数
int maxPoolThread = 100;
int totalThreadNum = dic.Count;
//当前正在运行的线程
var runingHt = new Dictionary<int, clsSubThread>();
//处于等待队列的未运行的线程
var unRunHt = new Dictionary<int, clsSubThread>();
//选取maxPoolThread个线程加入运行队列，其余放入未运行的等待队列
for (int i = 1; i <= totalThreadNum; i++)
{
clsSubThread th = new clsSubThread(i, dic[i]);
if (i <= maxPoolThread)
{
runingHt.Add(i,th);
th.Start();
}
else
{
unRunHt.Add(i, th);
}
}
while (true)
{
//初始化完成队列，用于存取已经执行完的线程的id
var stepFinishList = new List<int>();
//将完成的线程放入完成队列
foreach (int tid in runingHt.Keys)
{
var t = runingHt[tid];
if (t.IsStopped)
{
stepFinishList.Add(tid);
}
}
//1.遍历完成队列，从当前运行的线程队列中移除该线程
//2.对完成的线程执行回调，将数据持久化到数据库
//3.如果等待队列中还有数据，获取等待队列中的第一个，并执行该线程，将该线程从等待队列移除，加入到运行队列
foreach (int tid in stepFinishList)
{
Thread t1 = new Thread(new ParameterizedThreadStart(SaveOrUpdate));
t1.Start(runingHt[tid].ReturnList);
runingHt.Remove(tid);
if (unRunHt.Count > 0)
{
clsSubThread unRunThread = unRunHt.First().Value;
var unRunTid= unRunHt.First().Key;
unRunThread.Start();
runingHt.Add(unRunTid, unRunThread);
unRunHt.Remove(unRunTid);
}
}
//所有线程都完成后，跳出循环
if (runingHt.Count == 0 && unRunHt.Count == 0)
{
break;
}
}
return ratingClassPaimingStates;
}
///线程完成之后的回调动作，将返回的List保存到数据库
private void SaveOrUpdate(object o)
{
IList list = (IList)o;
Dao.SaveOrUpdateAll(list);
}
protected override void Page_Load(object sender, EventArgs e)
{
base.Page_Load(sender, e);
}
///click事件抓取网页信息，通过多线程协作插入到数据表
protected void Button1_Click(object sender, EventArgs e)
{
GetRatingInfo();
}
}
}
2.封装的线程类
using System.Collections.Generic;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using Contract.Domain;
using System.Web;
namespace ThreadTemplate
{
using System;
using System.Threading;
using System.IO;
/// <summary>
/// Summary description for clsSubThread.
/// </summary>
public class clsSubThread : IDisposable
{
private Thread thdSubThread = null;
private Mutex mUnique = new Mutex();
private bool blnIsStopped;
private bool blnSuspended;
private bool blnStarted;
private IList<CcrCompanyFundamental> clist;
private int threadId;
public int ThreadId { get { return threadId; } set { threadId = value; } }
private IList<CcrCreditScoreInfo> paiming;
public bool IsStopped
{
get { return blnIsStopped; }
}
public bool IsSuspended
{
get { return blnSuspended; }
}
public IList<CcrCreditScoreInfo> ReturnList
{
get { return paiming; }
}
public IList<CcrCompanyFundamental> CList
{
set { clist = value; }
}
public clsSubThread(int key,IList<CcrCompanyFundamental> pclist)
{
threadId = key;
paiming=new List<CcrCreditScoreInfo>();
//
// TODO: Add constructor logic here
//
blnIsStopped = true;
blnSuspended = false;
blnStarted = false;
clist = pclist;
}
/// <summary>
/// Start sub-thread
/// </summary>
public void Start()
{
if (!blnStarted)
{
thdSubThread = new Thread(new ThreadStart(SubThread));
blnIsStopped = false;
blnStarted = true;
thdSubThread.Start();
}
}
/// <summary>
/// Thread entry function 线程执行方法，从网站中用正则表达式，抓取需要的数据
/// </summary>
private void SubThread()
{
paiming = new List<CcrCreditScoreInfo>();
WebClient wc=new WebClient();
// do
{
for (int i = 0; i < clist.Count; i++)
{
CcrCompanyFundamental company = clist[i];
CcrCreditScoreInfo c = new CcrCreditScoreInfo();
c.FundamentalId = int.Parse(company.Id);
c.CompanyName = company.CompanyName;
string keyword = "湖南艾华集团股份有限公司";
using (Stream stream = wc.OpenRead("http://bgcheck.cn/MemberCenter/FirmCredit/Search.html?Keywords=" + keyword))
{
using (StreamReader sr = new StreamReader(stream, Encoding.UTF8))
{
string content = sr.ReadToEnd();
string ratingClasspatern = @"(?<=\[信用等级：([\s\S]*?)<a(.*)?[^>]*?>)([\s\S]*?)(?=</a>)";
string ratingSequencepatern = @"(?<=信用排名：([\s\S]*?)<span(.*)?[^>]*?>)([\s\S]*?)(?=</span>)";
string ratingStatepatern = @"(?<=信用状况：([\s\S]*?)<span(.*)?[^>]*?>)([\s\S]*?)(?=</span>)";
MatchCollection ratingClassmatches = Regex.Matches(content, ratingClasspatern);
MatchCollection ratingSequencematches = Regex.Matches(content, ratingSequencepatern);
MatchCollection ratingStatematches = Regex.Matches(content, ratingStatepatern);
string ratingClass = string.Empty;
string ratingSequence = string.Empty;
string ratingState = string.Empty;
foreach (Match match in ratingClassmatches)
{
ratingClass = match.Groups[0].Value;
break;
}
foreach (Match match in ratingSequencematches)
{
ratingSequence = match.Groups[0].Value;
break;
}
foreach (Match match in ratingStatematches)
{
ratingState = match.Groups[0].Value;
break;
}
c.RatingClass = ratingClass;
c.RatingSequence = ratingSequence;
c.RatingState = ratingState;
paiming.Add(c);
}
}
}
// Release CPU here
}
this.Stop();
//while (blnIsStopped == false);
}
/// <summary>
/// Suspend sub-thread
/// </summary>
public void Suspend()
{
if (blnStarted && !blnSuspended)
{
blnSuspended = true;
mUnique.WaitOne();
}
}
/// <summary>
/// Resume sub-thread
/// </summary>
public void Resume()
{
if (blnStarted && blnSuspended)
{
blnSuspended = false;
mUnique.ReleaseMutex();
}
}
/// <summary>
/// Stop sub-thread
/// </summary>
public void Stop()
{
if (blnStarted)
{
if (blnSuspended)
Resume();
blnStarted = false;
blnIsStopped = true;
thdSubThread.Join();
}
}
#region IDisposable Members
/// <summary>
/// Class resources dispose here
/// </summary>
public void Dispose()
{
// TODO: Add clsSubThread.Dispose implementation
Stop();//Stop thread first
GC.SuppressFinalize(this);
}
#endregion
}
}
