多线程采集器的核心代码
自己写的一个采集程序，用来采集小说，将线程控制的代码发出来。
通过采集的数据数量来分配线程数，可以限制最大线程数量
采集数据采取自动分配的方式，为每个线程平均分配一定量的采集任务，具体采集部分用WebClient或者其他方式都可
ManualResetEvent 用来控制线程的挂起和恢复
通过建立委托来实现进度条的刷新和控件提示文本的刷新
1 using System;
2 using System.Collections.Generic;
3 using System.Drawing;
4 using System.Windows.Forms;
5
6 using System.Threading;
7 using System.Text.RegularExpressions;
8
9 namespace GUIDemo
10 {
11 public partial class MainForm : Form
12 {
13 public MainForm()
14 {
15 InitializeComponent();
16 }
17
18 ManualResetEvent mReset = new ManualResetEvent(true);
19
20 List<Thread> threads = new List<Thread>();
21
22 void BtnStartClick(object sender, EventArgs e)
23 {
24 if(workStatus == WorkStatus.Pause)
25 {
26 if(threads.Count>0)
27 {
28 mReset.Set();
29 }
30 workStatus = WorkStatus.Working;
31 SetButtonStatus();
32 return;
33 }
34
35 workStatus = WorkStatus.Working;
36 SetButtonStatus();
37
38 List<ALink> links = new List<ALink>();
39
40 for(int i=0;i<76;i++)
41 {
42 links.Add(new ALink(i.ToString()));
43 }
44
45 progressBar1.Maximum = 76;
46 progressBar1.Value = 0;
47 th_cnt = 0;
48
49 int dataCount = links.Count;
50 int perCount = 25;
51
52 int threadCount = dataCount % perCount ==0? dataCount/perCount:dataCount/perCount+1;
53
54 int limitCount = 3;
55 //限制最大线程数量
56 if(threadCount>limitCount)
57 {
58 threadCount = limitCount;
59 perCount = dataCount % threadCount==0?dataCount/threadCount:dataCount/threadCount+1;
60 }
61
62 int cnt = 0; int i_cnt=0;
63
64 List<ALink> perLinks = new List<ALink>();
65 //分配资源
66 foreach(ALink link in links)
67 {
68 if(cnt%perCount==0&&cnt>0)
69 {
70 Thread thread = new Thread(new ParameterizedThreadStart(DoWork));
71 thread.Name = "thread"+i_cnt.ToString();
72 thread.Start(Clone(perLinks));
73 threads.Add(thread);
74 i_cnt++;
75 perLinks.Clear();
76 }
77 perLinks.Add(link);
78 cnt++;
79 }
80
81 if(perLinks.Count>0&&i_cnt<threadCount)
82 {
83 Thread thread = new Thread(new ParameterizedThreadStart(DoWork));
84 thread.Name = "thread"+i_cnt.ToString();
85 thread.Start(Clone(perLinks));
86 threads.Add(thread);
87 i_cnt++;
88 perLinks.Clear();
89 }
90
91 //Thread.Sleep(1000);
92 //flagExit = true;
93 }
94
95 public static bool flagExit = false;
96
97 static object locker = new object();
98
99 static int th_cnt = 0;
100
101 void DoWork(object threadParam)
102 {
103 List<ALink> items = threadParam as List<ALink>;
104 foreach (ALink link in items.ToArray())
105 {
106 mReset.WaitOne();
107 if(flagExit) break;
108 lock (locker) {
109 th_cnt++;
110 UpdateProcess(th_cnt);
111 }
112 OutputText(Thread.CurrentThread.Name+":"+link.Url);
113 ShowText(link.Url);
114
115 Thread.Sleep(50);
116 }
117 if(flagExit)
118 {
119 OutputText(Thread.CurrentThread.Name+":exit");
120 }else{
121 OutputText(Thread.CurrentThread.Name+":finlished");
122 }
123 OutputText(th_cnt.ToString());
124 }
125
126 delegate void ControlHandler(string text);
127 delegate void ProcessBarHandler(int i);
128
129 void ShowText(string text)
130 {
131 if(!label1.InvokeRequired)
132 {
133 label1.Text = text;
134 }
135 else
136 {
137 ControlHandler show = new ControlHandler(ShowText);
138 BeginInvoke(show,new object[]{text});
139 }
140 }
141
142 void UpdateProcess(int i)
143 {
144 if(!progressBar1.InvokeRequired)
145 {
146 progressBar1.Value = i>progressBar1.Maximum ? progressBar1.Maximum:i;
147 }
148 else
149 {
150 ProcessBarHandler show = new ProcessBarHandler(UpdateProcess);
151 BeginInvoke(show,new object[]{i});
152 }
153 }
154
155 static List<T> Clone<T>(IEnumerable<T> oldList)
156 {
157 return new List<T>(oldList);
158 }
159
160 void OutputText(string text)
161 {
162 System.Diagnostics.Debug.WriteLine(text);
163 }
164
165 void ReleaseThread()
166 {
167 if(threads.Count>0)
168 {
169 foreach(Thread th in threads)
170 {
171 th.Abort();
172 }
173 threads.Clear();
174 }
175 }
176
177 void MainFormFormClosing(object sender, FormClosingEventArgs e)
178 {
179 ReleaseThread();
180 }
181
182 void BtnStopClick(object sender, EventArgs e)
183 {
184 if(workStatus== WorkStatus.Working || workStatus== WorkStatus.Pause)
185 {
186 ReleaseThread();
187
188 progressBar1.Value = 0;
189 progressBar1.Refresh();
190
191 th_cnt = 0;
192
193 workStatus = WorkStatus.Stop;
194 SetButtonStatus();
195 }
196 }
197
198 WorkStatus workStatus = WorkStatus.Waitting;
199
200 void BtnPauseClick(object sender, EventArgs e)
201 {
202 if(workStatus == WorkStatus.Working)
203 {
204 if(threads.Count>0)
205 {
206 mReset.Reset();
207 workStatus = WorkStatus.Pause;
208 SetButtonStatus();
209 }
210 }
211 }
212
213 void SetButtonStatus()
214 {
215 switch(workStatus)
216 {
217 case WorkStatus.Working:
218 btnStart.Enabled = false;
219 btnPause.Enabled = true;
220 btnStop.Enabled = true;
221 break;
222 case WorkStatus.Pause:
223 btnStart.Enabled = true;
224 btnPause.Enabled = false;
225 btnStop.Enabled = true;
226 break;
227 case WorkStatus.Stop:
228 btnStart.Enabled = true;
229 btnPause.Enabled = false;
230 btnStop.Enabled = false;
231 break;
232 case WorkStatus.Waitting:
233 btnStart.Enabled = true;
234 btnPause.Enabled = false;
235 btnStop.Enabled = false;
236 break;
237 }
238 }
239
240 void MainFormLoad(object sender, EventArgs e)
241 {
242 SetButtonStatus();
243 }
244 }
245
246 public class ALink
247 {
248 public string Url="";
249 public ALink(string url)
250 {
251 this.Url = url;
252 }
253 }
254
255 enum WorkStatus
256 {
257 Pause,Stop,Working,Waitting
258 }
259 }
2 using System.Collections.Generic;
3 using System.Drawing;
4 using System.Windows.Forms;
5
6 using System.Threading;
7 using System.Text.RegularExpressions;
8
9 namespace GUIDemo
10 {
11 public partial class MainForm : Form
12 {
13 public MainForm()
14 {
15 InitializeComponent();
16 }
17
18 ManualResetEvent mReset = new ManualResetEvent(true);
19
20 List<Thread> threads = new List<Thread>();
21
22 void BtnStartClick(object sender, EventArgs e)
23 {
24 if(workStatus == WorkStatus.Pause)
25 {
26 if(threads.Count>0)
27 {
28 mReset.Set();
29 }
30 workStatus = WorkStatus.Working;
31 SetButtonStatus();
32 return;
33 }
34
35 workStatus = WorkStatus.Working;
36 SetButtonStatus();
37
38 List<ALink> links = new List<ALink>();
39
40 for(int i=0;i<76;i++)
41 {
42 links.Add(new ALink(i.ToString()));
43 }
44
45 progressBar1.Maximum = 76;
46 progressBar1.Value = 0;
47 th_cnt = 0;
48
49 int dataCount = links.Count;
50 int perCount = 25;
51
52 int threadCount = dataCount % perCount ==0? dataCount/perCount:dataCount/perCount+1;
53
54 int limitCount = 3;
55 //限制最大线程数量
56 if(threadCount>limitCount)
57 {
58 threadCount = limitCount;
59 perCount = dataCount % threadCount==0?dataCount/threadCount:dataCount/threadCount+1;
60 }
61
62 int cnt = 0; int i_cnt=0;
63
64 List<ALink> perLinks = new List<ALink>();
65 //分配资源
66 foreach(ALink link in links)
67 {
68 if(cnt%perCount==0&&cnt>0)
69 {
70 Thread thread = new Thread(new ParameterizedThreadStart(DoWork));
71 thread.Name = "thread"+i_cnt.ToString();
72 thread.Start(Clone(perLinks));
73 threads.Add(thread);
74 i_cnt++;
75 perLinks.Clear();
76 }
77 perLinks.Add(link);
78 cnt++;
79 }
80
81 if(perLinks.Count>0&&i_cnt<threadCount)
82 {
83 Thread thread = new Thread(new ParameterizedThreadStart(DoWork));
84 thread.Name = "thread"+i_cnt.ToString();
85 thread.Start(Clone(perLinks));
86 threads.Add(thread);
87 i_cnt++;
88 perLinks.Clear();
89 }
90
91 //Thread.Sleep(1000);
92 //flagExit = true;
93 }
94
95 public static bool flagExit = false;
96
97 static object locker = new object();
98
99 static int th_cnt = 0;
100
101 void DoWork(object threadParam)
102 {
103 List<ALink> items = threadParam as List<ALink>;
104 foreach (ALink link in items.ToArray())
105 {
106 mReset.WaitOne();
107 if(flagExit) break;
108 lock (locker) {
109 th_cnt++;
110 UpdateProcess(th_cnt);
111 }
112 OutputText(Thread.CurrentThread.Name+":"+link.Url);
113 ShowText(link.Url);
114
115 Thread.Sleep(50);
116 }
117 if(flagExit)
118 {
119 OutputText(Thread.CurrentThread.Name+":exit");
120 }else{
121 OutputText(Thread.CurrentThread.Name+":finlished");
122 }
123 OutputText(th_cnt.ToString());
124 }
125
126 delegate void ControlHandler(string text);
127 delegate void ProcessBarHandler(int i);
128
129 void ShowText(string text)
130 {
131 if(!label1.InvokeRequired)
132 {
133 label1.Text = text;
134 }
135 else
136 {
137 ControlHandler show = new ControlHandler(ShowText);
138 BeginInvoke(show,new object[]{text});
139 }
140 }
141
142 void UpdateProcess(int i)
143 {
144 if(!progressBar1.InvokeRequired)
145 {
146 progressBar1.Value = i>progressBar1.Maximum ? progressBar1.Maximum:i;
147 }
148 else
149 {
150 ProcessBarHandler show = new ProcessBarHandler(UpdateProcess);
151 BeginInvoke(show,new object[]{i});
152 }
153 }
154
155 static List<T> Clone<T>(IEnumerable<T> oldList)
156 {
157 return new List<T>(oldList);
158 }
159
160 void OutputText(string text)
161 {
162 System.Diagnostics.Debug.WriteLine(text);
163 }
164
165 void ReleaseThread()
166 {
167 if(threads.Count>0)
168 {
169 foreach(Thread th in threads)
170 {
171 th.Abort();
172 }
173 threads.Clear();
174 }
175 }
176
177 void MainFormFormClosing(object sender, FormClosingEventArgs e)
178 {
179 ReleaseThread();
180 }
181
182 void BtnStopClick(object sender, EventArgs e)
183 {
184 if(workStatus== WorkStatus.Working || workStatus== WorkStatus.Pause)
185 {
186 ReleaseThread();
187
188 progressBar1.Value = 0;
189 progressBar1.Refresh();
190
191 th_cnt = 0;
192
193 workStatus = WorkStatus.Stop;
194 SetButtonStatus();
195 }
196 }
197
198 WorkStatus workStatus = WorkStatus.Waitting;
199
200 void BtnPauseClick(object sender, EventArgs e)
201 {
202 if(workStatus == WorkStatus.Working)
203 {
204 if(threads.Count>0)
205 {
206 mReset.Reset();
207 workStatus = WorkStatus.Pause;
208 SetButtonStatus();
209 }
210 }
211 }
212
213 void SetButtonStatus()
214 {
215 switch(workStatus)
216 {
217 case WorkStatus.Working:
218 btnStart.Enabled = false;
219 btnPause.Enabled = true;
220 btnStop.Enabled = true;
221 break;
222 case WorkStatus.Pause:
223 btnStart.Enabled = true;
224 btnPause.Enabled = false;
225 btnStop.Enabled = true;
226 break;
227 case WorkStatus.Stop:
228 btnStart.Enabled = true;
229 btnPause.Enabled = false;
230 btnStop.Enabled = false;
231 break;
232 case WorkStatus.Waitting:
233 btnStart.Enabled = true;
234 btnPause.Enabled = false;
235 btnStop.Enabled = false;
236 break;
237 }
238 }
239
240 void MainFormLoad(object sender, EventArgs e)
241 {
242 SetButtonStatus();
243 }
244 }
245
246 public class ALink
247 {
248 public string Url="";
249 public ALink(string url)
250 {
251 this.Url = url;
252 }
253 }
254
255 enum WorkStatus
256 {
257 Pause,Stop,Working,Waitting
258 }
259 }
