1 #ifndef _Crawl_H_031104_
2 #define _Crawl_H_031104_
3
4 //#include <openssl/md5.h>
5 #include <zlib.h>
6
7 #include "Tse.h"
8 #include "Http.h"
9 #include "StrFun.h"
10 #include "Url.h"
11 #include "Page.h"
12 #include "TianwangFile.h"
13 #include "IsamFile.h"
14 #include "Link4SEFile.h"
15
16 using namespace std;
17
18 class CCrawl
19 {
20 public:
21 string m_sInputFileName; //种子URL的文件名字: tse_seed.pku
22 string m_sOutputFileName; //保存我们已经访问过的URL的文件名字: visited.all
23
24 CIsamFile m_isamFile; // ISAM file handle
25
26 ofstream m_ofsVisitedUrlFile; //visited.all的文件句柄
27 ofstream m_ofsLink4SEFile; //link4SE.url的文件句柄
28 ofstream m_ofsLink4HistoryFile; //link4History.url的文件句柄
29 ofstream m_ofsUnreachHostFile; //tse_unreachHost.list的文件句柄
30
31 ofstream m_ofsVisitedUrlMD5File;//tse_md5.visitedurl的文件句柄
32 ofstream m_ofsVisitedPageMD5File;//tse_md5.visitedpage
33
34 ofstream m_ofsUnreachUrlFile; // unreach URL file handle
35
36
37 public:
38 CCrawl();//无参构造函数 "tse_seed.pku" "visited.all"
39 CCrawl(string strInputFile, string strOutputFile);
40 ~CCrawl();
41
42 //CCrawl类中最重要的函数
43 void DoCrawl();
44
45 //根据URL以及套接字文件描述符抓取URL对应的网页
46 void DownloadFile( CTianwangFile *pTianwangFile,CLink4SEFile *pLink4SEFile, CUrl iUrl, int& nGSock);
47
48 //每个线程函数start()都调用这个函数
49 void fetch(void *arg);
50
51 //如果url满足条件加到mmapUrls[待访问的url]容器中
52 void AddUrl(const char *url);
53
54 void GetVisitedUrlMD5();//得到已经访问过的URL对应的MD5值，放入open list[setVisitedUrlMD5]中
55 void GetVisitedPageMD5();//得到已经访问过的web网页体对应的MD5值，放入setVisitedPageMD5中
56
57 void GetIpBlock();//得到阻塞的IP，放入mapIpBlock容器中
58
59 void GetUnreachHostMD5();//得到不可到达的主机号,放入setUnreachHostMD5中
60 void OpenFilesForOutput();//打开所有的输出流
61
62 // save in the process
63 void SaveTianwangRawData(CTianwangFile *pTianwangFile,CUrl *pUrl, CPage *pPage);//将抓取的网页以天网格式存储
64 void SaveLink4SERawData(CLink4SEFile *pLink4SEFile,CUrl *pUrl, CPage *pPage);//将抓取的网页从中提取超链接信息建立网页结构库
65
66 void SaveIsamRawData(CUrl *pUrl, CPage *Page);
67 void SaveVisitedUrl(string url);//保存已经访问过的URL
68 void SaveUnreachHost(string host);//保存不可到达的主机号
69 void SaveLink4SE(CPage *Page);//保存为搜索引擎准备的超链接信息
70 bool SaveLink4SE031121(void *arg);
71 void SaveLink4History(CPage *Page)//保存为历史网页存档准备的超链接信息
72
73 // save while the program running
74 void SaveVisitedUrlMD5(string md5);//保存已经访问过的URL对应的MD5值
75 void SaveVisitedPageMD5(string md5);//得到已经访问过的web网页体对应的MD5值
76
77 };
78
79 #endif /* _CRAWL_H_031104_ */
1 #include "Crawl.h"
2 #include "Url.h"
3 #include "Md5.h"
4
5 #include <list.h>
6 #include <hlink.h>
7 #include <uri.h>
8
9 extern pthread_mutex_t mymutex;
10 extern map<string, string> mapCacheHostLookup; //DNS缓存
11 extern vector<string> vsUnreachHost;
12 extern char **ParseRobot(char *data, char len);
13
14 set<string> setVisitedUrlMD5; //open list[已经访问的URL对应的MD5值]
15 set<string> setVisitedPageMD5; //已经访问过的web网页体对应的MD5值
16 set<string> setUnvisitedUrlMD5; //close list[没有访问过的URL对应的MD5值]
17
18 set<string> setUnreachHostMD5; //不可达到的主机号对应的MD5值的集合
19
20 multimap<string, string, less<string> > replicas; //web网页体对应的MD5值<->web网页体对应的URL
21
22 //定义线程的互斥变量并初始化
23 pthread_mutex_t mutexCollection = PTHREAD_MUTEX_INITIALIZER; //保护mmapUrls资源
24 pthread_mutex_t mutexUnreachHost = PTHREAD_MUTEX_INITIALIZER; //保护setUnreachHostMD5&&m_ofsUnreachHostFile资源
25 pthread_mutex_t mutexUnvisitedUrlMD5 = PTHREAD_MUTEX_INITIALIZER; //保护setUnvisitedUrlMD5资源
26 pthread_mutex_t mutexVisitedUrlMD5 = PTHREAD_MUTEX_INITIALIZER; //保护setVisiteUrlMD5&&m_ofsVisiteUrlMD5File资源
27 pthread_mutex_t mutexVisitedPageMD5 = PTHREAD_MUTEX_INITIALIZER; //保护setVisitePageMD5&&m_ofsVisitePageMD5File资源
28
29 pthread_mutex_t mutexDetect = PTHREAD_MUTEX_INITIALIZER;
30 pthread_mutex_t mutexLink4SEFile = PTHREAD_MUTEX_INITIALIZER; //保护m_ofsLink4SEFile资源
31 pthread_mutex_t mutexLink4HistoryFile = PTHREAD_MUTEX_INITIALIZER;
32 pthread_mutex_t mutexIsamFile = PTHREAD_MUTEX_INITIALIZER;
33 pthread_mutex_t mutexVisitedUrlFile = PTHREAD_MUTEX_INITIALIZER;
34 pthread_mutex_t mutexUnreachHostFile = PTHREAD_MUTEX_INITIALIZER;
35 pthread_mutex_t mutexReplicas = PTHREAD_MUTEX_INITIALIZER;
36 //pthread_mutex_t mutexMemory = PTHREAD_MUTEX_INITIALIZER;
37
38 map<unsigned long, unsigned long> mapIpBlock; //IP阻塞范围
39 bool b_fOver; //线程运行控制参数
40 //multimap<string,string, less<string> > mmapUrls;
41 multimap<string, string> mmapUrls; //保存没有访问过的URL的主机号<->对应的URL
42
43 typedef map<unsigned long, unsigned long>::value_type valTypeIpBlock;
44 typedef map<string, string>::value_type mvalType;
45
46 void SaveReplicas(const char* filename); //保存镜像网页对应的URL的一个值到指定的文件名中
47
48 struct package {
49 CCrawl *crawl;
50 CPage *page;
51 };
52
53 vector<string> vsParsedLinks;
54
55 int onfind(const char *elem, const char *attr, struct uri *uri, void *arg) {
56 struct package *p = (struct package*) arg;
57 char buff[URL_LEN + 1];
58
59 if (uri_recombine(uri, buff, URL_LEN + 1,
60 C_SCHEME | C_AUTHORITY | C_PATH | C_QUERY) >= 0)
61
62 {
63 vsParsedLinks.push_back(buff);
64 if (!p->page->IsFilterLink(buff)) {
65 // accept "a,link,frame,iframe,img,area"
66
67 if (strcasecmp(elem, "img") == 0) {
68 pthread_mutex_lock(&mutexLink4HistoryFile);
69 if (p->crawl->m_ofsLink4HistoryFile) {
70 p->crawl->m_ofsLink4HistoryFile << buff << endl;
71 }
72 pthread_mutex_unlock(&mutexLink4HistoryFile);
73
74 } else {
75 p->crawl->AddUrl(buff);
76 }
77 /*
78 else if (strcasecmp(elem, "img") == 0)
79 {
80 pthread_mutex_lock(&mutexLink4HistoryFile);
81 if( p->crawl->m_ofsLink4HistoryFile ){
82 p->crawl->m_ofsLink4HistoryFile << p->page->m_sUrl << endl;;
83 }
84 pthread_mutex_unlock(&mutexLink4HistoryFile);
85 }
86 */
87 }
88 }
89
90 uri_destroy(uri);
91 free(uri);
92 return 1;
93 }
94
95 /***********************************************************************
96 * Function name: start
97 * Input argv:
98 * -- arg: the CCrawl handle
99 * Output argv:
100 * --
101 * Return:
102 ***********************************************************************/
103 //线程函数-->每个线程函数调用fetch(void*arg)函数
104 void* start(void *arg) {
105 ((CCrawl*) arg)->fetch(arg);
106 }
107
108 /*
109 这个函数设计的很巧妙,这里说的巧妙不是函数写有多hi：
110 我们知道spider在最开始抓取网页的时候需要种子url,
111 我们这个spider的种子url文件库是tse_seed.ur文件
112 而这个函数正好在我们强制中断程序的时候，将mmapUrls
113 中没有访问完的url放入tse_unvisited.url文件中,
114 扩充了我们的种子URL库!*/
115 void SaveUnvisitedUrl() {
116 ofstream ofsUnvisitedUrl;
117 ofsUnvisitedUrl.open(UNVISITED_FILE.c_str(),
118 ios::in | ios::out | ios::trunc | ios::binary); //以二进制可追加写方式打开文件
119 if (!ofsUnvisitedUrl) //打开失败
120 {
121 cerr << "cannot open " << UNVISITED_FILE << "for output" << endl;
122 exit(-1);
123 }
124
125 //将mmapUrls中没有访问完的url放入tse_unvisited.url文件中,扩充了我们的URL种子库!
126 multimap<string, string>::iterator it = mmapUrls.begin();
127 for (; it != mmapUrls.end(); it++) {
128 ofsUnvisitedUrl << ((*it).second).c_str() << "\n";
129 }
130
131 ofsUnvisitedUrl << endl;
132 ofsUnvisitedUrl.close();
133
134 }
135
136 /***********************************************************************
137 * Function name: fetch
138 * Input argv:
139 * -- arg: the CCrawl handle
140 * Output argv:
141 * --
142 * Return:
143 ***********************************************************************/
144 void CCrawl::fetch(void *arg) //每个线程都执行这个函数
145 {
146 string strUrl, host;
147
148 int nGSock = -1; //之前的套接字文件描述符
149 string strGHost = ""; //字前的主机号
150
151 // create a Tianwang file for output the raw page data
152 string ofsName = DATA_TIANWANG_FILE + "." + CStrFun::itos(pthread_self()); //Tianwang.raw+"线程号"
153 CTianwangFile tianwangFile(ofsName); //创建一个天网格式的文件，保存为原始网页库
154
155 // create a Link4SE file for output the raw link data
156 ofsName = DATA_LINK4SE_FILE + "." + CStrFun::itos(pthread_self()); //Link4SE.raw+"线程号"
157 CLink4SEFile link4SEFile(ofsName); //创建一个网页结构库
158
159 int iSleepCnt = 0; //线程运行控制参数
160 for (;;) {
161 pthread_mutex_lock(&mutexCollection); //互斥的锁定函数
162 //if( !mmapUrls.empty() ){
163 int cnt = mmapUrls.size();
164 if (cnt > 0) {
165 //已经收集的没有访问的url
166 cout << "collection has: " << cnt << " unvisited urls" << endl;
167 multimap<string, string>::iterator it = mmapUrls.begin();
168 if (it != mmapUrls.end()) {
169 // get an URL
170 strUrl = (*it).second; //从待访问的URL队列中得到一个URL进行访问
171
172 // remove it from the collection
173 mmapUrls.erase(it); //删除迭代器所指的元素
174
175 pthread_mutex_unlock(&mutexCollection); //互斥的解锁函数
176
177 // parse URL
178 CUrl iUrl; //关键是看看strUrl是否有http://协议号,没有返回false
179 if (iUrl.ParseUrlEx(strUrl) == false) {
180 cout << "ParseUrlEx error in fetch(): " << strUrl << endl;
181 continue;
182 }
183
184 //表明现在抓取的网页所在的主机，同之前抓取的网页所在的主机不同
185 //故,我们不能利用之前的套接字文件描述符进行CS通信,必须创建新的
186 //套接字文件描述符进行通信,这是由于循环导致的
187 if (strGHost != iUrl.m_sHost) {
188 close(nGSock);
189 nGSock = -1;
190 strGHost = iUrl.m_sHost;
191 }
192
193 //根据URL以及套接字文件描述符抓取URL对应的网页,并保存为原始网页库和网页结构库
194 ((CCrawl*) arg)->DownloadFile(&tianwangFile, &link4SEFile, iUrl,
195 nGSock);
196
197 cnt = 0;
198 } else {
199 pthread_mutex_unlock(&mutexCollection);
200 }
201 } else {
202 //待访问的URL队列mmapUrls中没有URL了，这个时候我们必须挂起线程进行等待
203 pthread_mutex_unlock(&mutexCollection);
204 usleep(1000);
205 iSleepCnt++;
206 }
207
208 if (b_fOver == true && iSleepCnt == 200) //当URL队列mmapUrls有200次都是空的时候就结束这个线程调用的fetch()函数
209 break;
210 /*
211 if( b_fOver == true ){
212 break;
213 } else if( cnt == 100 ) {
214 cout << "w.";
215 cnt = 0;
216 }
217 */
218 }
219
220 tianwangFile.Close();
221 link4SEFile.Close();
222 }
223
224 /***********************************************************************
225 * Function name: DownloadFile
226 * Input argv:
227 * -- pTianwang: the CCrawl handle
228 * -- pLink4SE: the CCrawl handle
229 * -- iUrl: the URL for crawling
230 * -- nGSock: the previous global socket
231 * Output argv:
232 * --
233 * Return:
234 ***********************************************************************/
235
236 //根据URL以及套接字文件描述符抓取URL对应的网页,并保存为原始网页库和网页结构库
237 void CCrawl::DownloadFile(CTianwangFile *pTianwangFile,
238 CLink4SEFile *pLink4SEFile, CUrl iUrl, int& nGSock) {
239 char *downloaded_file = NULL, //网页体信息
240 *fileHead = NULL, //网页头信息
241 *location = NULL; //网页的重定向信息
242 int file_length = 0; //网页体真实的字节长度
243 string strUrlLocation = ""; //保存网页的重定向超链接
244
245 //之后请求的网页和之前请求的网页位于同一个主机上，我们可以利用之前的套接字文件描述符进行通信，这样我们可以节约带宽，节省时间
246 int nSock = nGSock; //将之前的套接字文件描述符赋值给nSock
247
248 cout << "1. pid=" << pthread_self() << " sock = " << nGSock << endl;
249
250 CHttp http;
251
252 //这是一个真正的抓取网页的函数,有了URL搜集系统可以根据URL的标识抓取其对应的网页
253 file_length = http.Fetch(iUrl.m_sUrl, &downloaded_file, &fileHead,
254 &location, &nSock);
255
256 int nCount = 0; //用来标识URL重定向的次数,如果重定向了3次，我们就不要抓取它对应的网页
257
258 while (file_length == -300) //表明该iUrl.m_sUrl对应的网页重定向了
259 { // moved to an another place
260 if (strlen(location) > URL_LEN - 1 || nCount == 3
261 || strlen(location) == 0) {
262 if (location) {
263 //pthread_mutex_lock(&mutexMemory);
264 free(location);
265 location = NULL;
266 //pthread_mutex_unlock(&mutexMemory);
267 }
268 file_length = -1;
269 break;
270 }
271
272 //将获取到的重定向的URL给strUrlLocation为下次抓取网页做准备
273 strUrlLocation = location;
274 if (location) {
275 //pthread_mutex_lock(&mutexMemory);
276 free(location);
277 location = NULL;
278 //pthread_mutex_unlock(&mutexMemory);
279 }
280
281 //这个地方要注意,因为重定向的URL可能是相对路径，所以我们必须将它转化为绝对路径
282 //跟CPage类中提取超链接信息一样
283 string::size_type idx1 = CStrFun::FindCase(strUrlLocation, "http");
284
285 if (idx1 != 0) { //没有找"http://"协议号
286
287 char c1 = iUrl.m_sUrl.at(iUrl.m_sUrl.length() - 1);
288 char c2 = strUrlLocation.at(0);
289
290 if (c2 == '/') //重定向的URL一定是相对路径
291 {
292 strUrlLocation = "http://" + iUrl.m_sHost + strUrlLocation;
293 } else if (c1 != '/' && c2 != '/') {
294 string::size_type idx;
295
296 idx = iUrl.m_sUrl.rfind('/');
297 if (idx != string::npos) {
298 if (idx > 6) { // > strlen("http://..")
299 strUrlLocation = iUrl.m_sUrl.substr(0, idx + 1)
300 + strUrlLocation;
301 } else {
302 strUrlLocation = iUrl.m_sUrl + "/" + strUrlLocation;
303 }
304
305 } else {
306 file_length = -1;
307 break;
308 }
309 } else {
310 if (c1 == '/') {
311 strUrlLocation = iUrl.m_sUrl + strUrlLocation;
312 } else {
313 strUrlLocation = iUrl.m_sUrl + "/" + strUrlLocation;
314 }
315 }
316 }
317
318 CPage iPage;
319 if (iPage.IsFilterLink(strUrlLocation)) { //如果得到的重定向URL是要过滤的URL，我们立刻结束不再抓取
320 file_length = -1;
321 break;
322 }
323
324 cout << "2. pid=" << pthread_self() << " sock = " << nGSock << endl;
325 file_length = http.Fetch(strUrlLocation, &downloaded_file, &fileHead,
326 &location, &nSock);
327 nCount++;
328 }
329
330 nGSock = nSock; //将新得到的套接字文件描述符给之前的套接字文件描述符，为下次重用做准备
331
332 if (file_length == -1) { //其他的各种错误，这个错误的原因在http.Fetch()中
333 cout << "!-: " << iUrl.m_sUrl << endl;
334 //pthread_mutex_lock(&mutexMemory);
335 if (fileHead) {
336 free(fileHead);
337 fileHead = NULL;
338 }
339 if (downloaded_file) {
340 free(downloaded_file);
341 downloaded_file = NULL;
342 }
343 //pthread_mutex_unlock(&mutexMemory);
344
345 cout << "-unreach host: " << iUrl.m_sHost << endl;
346 ;
347 return;
348 }
349
350 if (file_length == -2) { // out of ip block .//在IP阻塞范围内
351 //pthread_mutex_lock(&mutexMemory);
352 if (fileHead) {
353 free(fileHead);
354 fileHead = NULL;
355 }
356 if (downloaded_file) {
357 free(downloaded_file);
358 downloaded_file = NULL;
359 }
360 //pthread_mutex_unlock(&mutexMemory);
361
362 // save unreach host
363 SaveUnreachHost(iUrl.m_sHost);
364
365 cout << "-out of block host: " << iUrl.m_sHost << endl;
366 ;
367 return;
368 }
369
370 if (file_length == -3) { // invalid host or ip//URL的主机号是无效的主机号
371 //pthread_mutex_lock(&mutexMemory);
372 if (fileHead) {
373 free(fileHead);
374 fileHead = NULL;
375 }
376 if (downloaded_file) {
377 free(downloaded_file);
378 downloaded_file = NULL;
379 }
380 //pthread_mutex_unlock(&mutexMemory);
381 cout << "-invalid host: " << iUrl.m_sHost << endl;
382 return;
383 }
384
385 if (file_length == -4) { // MIME is image/xxx//图片类型的网页
386 //pthread_mutex_lock(&mutexMemory);
387 if (fileHead) {
388 free(fileHead);
389 fileHead = NULL;
390 }
391 if (downloaded_file) {
392 free(downloaded_file);
393 downloaded_file = NULL;
394 }
395 //pthread_mutex_unlock(&mutexMemory);
396
397 if (m_ofsLink4HistoryFile) { //为历史网页存档准备的链接
398 pthread_mutex_lock(&mutexLink4HistoryFile);
399 m_ofsLink4HistoryFile << iUrl.m_sUrl << endl;
400 ; //将该URL保存在link4History.url文件中
401 pthread_mutex_unlock(&mutexLink4HistoryFile);
402 }
403
404 cout << "-imgage host: " << iUrl.m_sHost << endl;
405 return;
406 }
407
408 /* still experiment
409 char **dir;
410 dir = ParseRobot( downloaded_file, file_length);
411 for( int i = 0; dir[i] != NULL ; i++){
412 cout << dir[i] << endl;
413 free( dir[i] );
414 }
415
416 exit(1);
417 */
418
419 // so small, maybe some unuseful info, skipped
420 //if(file_length < 40){ // for ImgSE,
421 /*
422 if(file_length < 256){ // for SE
423 //pthread_mutex_lock(&mutexMemory);
424 if (fileHead)
425 {
426 free(fileHead); fileHead=NULL;
427 }
428 if (downloaded_file)
429 {
430 free(downloaded_file); downloaded_file=NULL;
431 }
432 //pthread_mutex_unlock(&mutexMemory);
433 cout << "#";
434 return;
435 }
436 */
437
438 // deal with normal page
439
440 //处理正常的网页[网页头信息和网页体信息只要有一个是NULL，我们就认为它不是正常的网页]
441 if (!fileHead || !downloaded_file) //不能获得网页头信息或者网页体信息
442 {
443 //pthread_mutex_lock(&mutexMemory);
444 if (fileHead) {
445 free(fileHead);
446 fileHead = NULL;
447 }
448 if (downloaded_file) {
449 free(downloaded_file);
450 downloaded_file = NULL;
451 }
452 //pthread_mutex_unlock(&mutexMemory);
453 close(nGSock);
454 nGSock = -1;
455 cout << "-size0 host: " << iUrl.m_sHost << endl;
456 return;
457 }
458
459 //这里很重要，将抓取到的网页信息全部放入CPage类中
460 CPage iPage(iUrl.m_sUrl, strUrlLocation, fileHead, downloaded_file,
461 file_length);
462 //pthread_mutex_lock(&mutexMemory);
463 if (fileHead) {
464 free(fileHead);
465 fileHead = NULL;
466 }
467 if (downloaded_file) {
468 free(downloaded_file);
469 downloaded_file = NULL;
470 }
471 //pthread_mutex_unlock(&mutexMemory);
472
473 //解析网页头信息
474 iPage.ParseHeaderInfo(iPage.m_sHeader);
475
476 if (iPage.m_bConnectionState == false) {
477 close(nGSock);
478 nGSock = -1;
479 }
480
481 // when crawling images for ImgSE, remember to comment the paragraph
482 // when crawling plain text for SE, remember to open the paragraph
483 // paragraph begin
484
485 // iPage.m_sContentType != "text/css" &&
486
487 //过滤掉不是我们想要的网页体的类型
488 if (iPage.m_sContentType != "text/html"
489 && iPage.m_sContentType != "text/plain"
490 && iPage.m_sContentType != "text/xml"
491 && iPage.m_sContentType != "application/msword"
492 && iPage.m_sContentType != "application/pdf"
493 && iPage.m_sContentType != "text/rtf"
494 && iPage.m_sContentType != "application/postscript"
495 && iPage.m_sContentType != "application/vnd.ms-execl"
496 && iPage.m_sContentType != "application/vnd.ms-powerpoint") {
497
498 cout << "-unwant type host: " << iUrl.m_sHost << endl;
499 return;
500 }
501
502 // paragraph end
503
504 //解压缩开始
505 //如果是gzip编码，要解压缩，然后提取超链接信息，现在门户网站的首页有增大趋势
506 //为了加快传输速度，通常采用gzip编码压缩后传输
507 char sUnzipContent[1024000]; //1000K<1M
508 int nUnzipLength = 0;
509 if (iPage.m_sContentEncoding == "gzip"
510 && iPage.m_sContentType == "text/html") {
511
512 gzFile zip;
513 //这是一个过渡文件，将没有解压缩的网页体信息放入到这个文件中
514 string ofsGzipName;
515
516 ofsGzipName = CStrFun::itos(pthread_self()) + ".gz";
517
518 //以二进制截断的方式打开文件
519 //ios::trunc 如果文件存在，则将文件长度截断为0，并清除文件的内容，如果文件不存在，则创建该文件
520 ofstream ofsDownloadFile(ofsGzipName.c_str(), ios::trunc | ios::binary);
521
522 cout << "file_length: " << file_length << endl;
523 ofsDownloadFile.write(iPage.m_sContent.c_str(), iPage.m_nLenContent);
524 ofsDownloadFile.close();
525
526 zip = gzopen(ofsGzipName.c_str(), "rb");
527 if (zip == NULL) {
528 cout << "Open zip file " << ofsGzipName.c_str() << " error."
529 << endl;
530 exit(-1);
531 }
532
533 //解压缩过程，将解压缩后的网页体信息放入到缓冲区域sUnzipContent
534 nUnzipLength = gzread(zip, sUnzipContent, 1024000);
535 if (nUnzipLength == -1) {
536 cout << "Read zip file " << ofsGzipName.c_str() << " error."
537 << endl;
538 exit(-1);
539 }
540
541 sUnzipContent[nUnzipLength] = 0;
542
543 gzclose(zip);
544
545 //将解压缩后的网页体信息覆盖原来的没有解压缩的网页体信息
546 //iPage.m_sContent.assign(sUnzipContent,nUnzipLength);
547 //iPage.m_nLenContent=nUnzipLength;
548 }
549 //解压缩结束
550
551 CMD5 iMD5;
552 string strDigest;
553
554 /////////////////////////////
555 // because we can make sure the url in the setVisitedUrlMd5
556 // is not same(we have check it before insert it to the collection),
557 // we intert it directly. however...
558 //iMD5.GenerateMD5( (unsigned char*)iPage.m_sUrl.c_str(), iPage.m_sUrl.length() );
559
560 //判断该URL是否在open list[setVisitedUrlMD5]中,在返回;不在加到open list中，并保存
561 iMD5.GenerateMD5((unsigned char*) iUrl.m_sUrl.c_str(),iUrl.m_sUrl.length());
562 strDigest = iMD5.ToString();
563
564 pthread_mutex_lock(&mutexVisitedUrlMD5);
565 if (setVisitedUrlMD5.find(strDigest) != setVisitedUrlMD5.end()) //已经抓取过了！
566 {
567 cout << "!vurl: "; //1.crawled already
568 pthread_mutex_unlock(&mutexVisitedUrlMD5);
569 return;
570 }
571
572 //不在setVisitedUrlMD5中，现在必须插入setVisitedUrlMD5中
573 //因为该URL现在已经访问过了
574 setVisitedUrlMD5.insert(strDigest);
575 SaveVisitedUrlMD5(strDigest);
576 pthread_mutex_unlock(&mutexVisitedUrlMD5);
577
578 /////////////////////////////
579 // whether it is a visited page
580 // for ImgSE, should comment this paragraph
581 // for SE, should uncomment this paragraph
582
583 // begin
584
585 //判断该网页体是否已经访问过,访问过返回,没有访问过加到setVisitedPageMD5集合中
586 iMD5.GenerateMD5((unsigned char*) iPage.m_sContent.c_str(),iPage.m_sContent.length());
587 strDigest = iMD5.ToString();
588 pthread_mutex_lock(&mutexVisitedPageMD5);
589 //网页体MD5同URL的关系插入到容器replicas中
590 replicas.insert(pair<string, string>(strDigest, iPage.m_sUrl));
591 if (setVisitedPageMD5.find(strDigest) != setVisitedPageMD5.end()) //在setVisitedPageMD5中:表明出现了镜像网页
592 {
593 cout << "!vpage: "; // crawled already
594 pthread_mutex_unlock(&mutexVisitedPageMD5);
595 return;
596 }
597 setVisitedPageMD5.insert(strDigest);
598
599 SaveVisitedPageMD5(strDigest);
600 pthread_mutex_unlock(&mutexVisitedPageMD5);
601
602 // end
603
604 cout << "+";
605
606 ////////////////////
607 // save as Tianwang format
608 //将抓取到的网页以天网格式放到原始网页库中
609 SaveTianwangRawData(pTianwangFile, &iUrl, &iPage);
610
611 ////////////////////
612 // save visited Urls
613 if (iPage.m_sLocation.length() < 1) {
614 SaveVisitedUrl(iUrl.m_sUrl);
615 } else {
616 SaveVisitedUrl(iPage.m_sLocation);
617 }
618
619 //return; // just crawl seeds
620
621 /////////////////////////////////////
622 // Parse hyperlinks
623 if (iPage.m_sContentType != "text/html") { // we can only find links in tex/html
624 return;
625 }
626
627 /*
628 if (iPage.ParseHyperLinks() == false){
629 return;
630 }
631
632 SaveLink4SE( &iPage);
633 SaveLink4History( &iPage);
634
635 map<string,string>::iterator it4SE = iPage.m_mapLink4SE.begin();
636 string str;
637 for( ; it4SE!= iPage.m_mapLink4SE.end(); ++it4SE ){
638 str = (*it4SE).first;
639 AddUrl( str.c_str() );
640
641 }
642 */
643 // using XIE Han's link parser
644
645 struct uri page_uri;
646 //FILE *tmp;
647
648 //tmp = tmpfile();
649
650 //fwrite(iPage.m_sContent.c_str(), iPage.m_nLenContent, 1, tmp);
651 //fseek(tmp, 0, SEEK_SET);
652 //fclose(tmp);
653
654 pthread_mutex_lock(&mutexDetect);
655
656 if (iPage.m_sLocation.empty()) {
657 uri_parse_string(iPage.m_sUrl.c_str(), &page_uri);
658 } else {
659 uri_parse_string(iPage.m_sLocation.c_str(), &page_uri);
660 }
661
662 struct package p = { this, &iPage };
663 //hlink_detect(tmp, &page_uri, onfind, &p);
664
665 hlink_detect_string(iPage.m_sContent.c_str(), &page_uri, onfind, &p);
666
667 struct file_arg pLinks = { &iUrl, &iPage };
668
669 SaveLink4SE031121(&pLinks);
670
671 // save as Link4SE format
672 //SaveLink4SERawData(pLink4SEFile, &iUrl, &iPage);
673
674 pthread_mutex_unlock(&mutexDetect);
675
676 uri_destroy(&page_uri);
677 cout << "Parse End......" << endl;
678
679 return;
680 }
681
682 void SaveReplicas(const char* filename) {
683 //ofstream ofs(filename, ios::out|ios::app);
684 ofstream ofs(filename, ios::out | ios::binary | ios::app);
685 if (!ofs) {
686 cout << "error open file " << endl;
687 }
688 string md5;
689
690 pthread_mutex_lock(&mutexReplicas);
691 multimap<string, string, less<string> >::const_iterator it;
692 ostringstream *oss = 0;
693 int i = 0;
694 for (it = replicas.begin(); it != replicas.end(); it++) {
695 if (!md5.empty() && md5 != it->first) {
696 if (i >= 2)
697 ofs << (*oss).str() << endl;
698 //pthread_mutex_lock(&mutexMemory);
699 delete (oss);
700 oss = new ostringstream;
701 //pthread_mutex_unlock(&mutexMemory);
702 (*oss) << it->first << endl;
703 i = 0;
704 md5 = it->first;
705 } else if (md5.empty()) {
706 md5 = it->first;
707 //pthread_mutex_lock(&mutexMemory);
708 oss = new ostringstream;
709 //pthread_mutex_unlock(&mutexMemory);
710 (*oss) << it->first << endl;
711 i = 0;
712 }
713 if (oss != 0)
714 (*oss) << it->second << endl;
715 i++;
716 }
717
718 pthread_mutex_unlock(&mutexReplicas);
719 }
720
721 ////////////////////////////////////////////////////////////////////////////
722 // Construction/Destruction
723 ////////////////////////////////////////////////////////////////////////////
724
725 CCrawl::CCrawl() {
726 }
727
728 CCrawl::CCrawl(string inputFileName, string outputFileName) {
729 m_sInputFileName = inputFileName;
730 m_sOutputFileName = outputFileName; // + ".txt"
731 }
732
733 CCrawl::~CCrawl() {
734 m_ofsVisitedUrlFile.close();
735 m_ofsLink4SEFile.close();
736 m_ofsLink4HistoryFile.close();
737 m_isamFile.Close();
738 m_ofsVisitedUrlMD5File.close();
739 m_ofsVisitedPageMD5File.close();
740 }
741
742 /*****************************************************************
743 ** Function name: SigTerm
744 ** Input argv:
745 ** --
746 ** Output argv:
747 ** --
748 ** Return:
749 ** Function Description: signal function
750 ** Version: 1.0
751 ** Be careful:
752 *****************************************************************/
753 static void SigTerm(int x) {//信号处理函数
754 SaveUnvisitedUrl();
755 SaveReplicas("repli");
756
757 cout << "Terminated!" << endl;
758 exit(0);
759 }
760
761 void CCrawl::GetVisitedUrlMD5() {//得到已经访问过的URL对应的MD5值，放入open list[setVisitedUrlMD5]中
762 ifstream ifsMD5(URL_MD5_FILE.c_str(), ios::binary);
763 if (!ifsMD5) {
764 //cerr << "did not find " << UrlMD5_FILE << " for iutput" << endl;
765 return;
766 }
767
768 string strMD5;
769 while (getline(ifsMD5, strMD5)) {
770 setVisitedUrlMD5.insert(strMD5);
771 }
772
773 ifsMD5.close();
774 cout << "got " << setVisitedUrlMD5.size() << " md5 values of visited urls"
775 << endl;
776 }
777
778 void CCrawl::GetVisitedPageMD5() {//得到已经访问过的web网页体对应的MD5值，放入setVisitedPageMD5中
779 ifstream ifsMD5(PAGE_MD5_FILE.c_str(), ios::binary);
780 if (!ifsMD5) {
781 //cerr << "did not find " << PageMD5_FILE << " for iutput" << endl;
782 return;
783 }
784
785 string strMD5;
786 while (getline(ifsMD5, strMD5)) {
787 setVisitedPageMD5.insert(strMD5);
788 }
789
790 ifsMD5.close();
791 cout << "got " << setVisitedPageMD5.size() << " md5 values of visited pages"
792 << endl;
793 }
794
795 void CCrawl::GetIpBlock() {//得到阻塞的IP，放入mapIpBlock容器中
796 ifstream ifsIpBlock(IP_BLOCK_FILE.c_str());
797 if (!ifsIpBlock) {
798 //cerr << "Cannot open " << IP_BLOCK_FILE << " for input." << endl;
799 return;
800 }
801 string strIpBlock;
802 while (getline(ifsIpBlock, strIpBlock)) {
803 if (strIpBlock[0] == '\0' || strIpBlock[0] == '#'
804 || strIpBlock[0] == '\n') {
805
806 continue;
807 }
808
809 char buf1[64], buf2[64];
810
811 buf1[0] = '\0';
812 buf2[0] = '\0';
813 sscanf(strIpBlock.c_str(), "%s %s", buf1, buf2);
814
815 mapIpBlock.insert(valTypeIpBlock(inet_addr(buf1), inet_addr(buf2)));
816 }
817 ifsIpBlock.close();
818
819 }
820
821 void CCrawl::GetUnreachHostMD5() {//得到不可到达的主机号,放入setUnreachHostMD5中
822 //vsUnreachHost.reserve(MAX_UNREACHABLE_HOST_NUM);
823 ifstream ifsUnreachHost(UNREACH_HOST_FILE.c_str());
824 if (!ifsUnreachHost) {
825 cerr << "Cannot open " << UNREACH_HOST_FILE << " for input." << endl;
826 return;
827 }
828
829 string strUnreachHost;
830 //int i=0;
831 while (getline(ifsUnreachHost, strUnreachHost)) {
832 if (strUnreachHost[0] == '\0' || strUnreachHost[0] == '#'
833 || strUnreachHost[0] == '\n') {
834
835 continue;
836 }
837
838 CStrFun::Str2Lower(strUnreachHost, strUnreachHost.size());
839 //vsUnreachHost.push_back(strUnreachHost);
840 CMD5 iMD5;
841 iMD5.GenerateMD5((unsigned char*) strUnreachHost.c_str(),
842 strUnreachHost.size());
843 string strDigest = iMD5.ToString();
844 setUnreachHostMD5.insert(strDigest);
845 //i++;
846 //if(i == MAX_UNREACHABLE_HOST_NUM) break;
847 }
848
849 ifsUnreachHost.close();
850
851 }
852
853 /**************************************************************************************
854 * Function name: SaveTianwangRawData
855 * Input argv:
856 * -- pTianwangFile: tianwang file handle
857 * -- pUrl: url
858 * -- pPage: web page
859 * Output argv:
860 * --
861 * Return:
862 * Function Description: save raw page data as tianwang file
863 **************************************************************************************/
864 void CCrawl::SaveTianwangRawData(CTianwangFile *pTianwangFile, CUrl *pUrl,CPage *pPage) {//将抓取的网页以天网格式存储
865 if (!pTianwangFile || !pUrl || !pPage) {
866 return;
867 }
868
869 file_arg arg;
870 arg.pUrl = pUrl;
871 arg.pPage = pPage;
872
873 // each thread writes itself, so dnnot need mutex
874 pTianwangFile->Write((void*) &arg);
875 }
876
877 /**************************************************************************************
878 * Function name: SaveLink4SERawData
879 * Input argv:
880 * -- pLink4SEFile: link4SE file handle
881 * -- pUrl: url
882 * -- pPage: web page
883 * Output argv:
884 * --
885 * Return:
886 * Function Description: save raw page data as tianwang file
887 **************************************************************************************/
888 void CCrawl::SaveLink4SERawData(CLink4SEFile *pLink4SEFile, CUrl *pUrl,
889 CPage *pPage) { //将抓取的网页从中提取超链接信息建立网页结构库
890 if (!pLink4SEFile || !pUrl || !pPage) {
891 return;
892 }
893
894 file_arg arg;
895 arg.pUrl = pUrl;
896 arg.pPage = pPage;
897
898 // each thread writes itself, so dnnot need mutex
899 pLink4SEFile->Write((void*) &arg);
900 }
901
902 /**************************************************************************************
903 * Function name: SaveIsamRawData
904 * Input argv:
905 * -- pUrl: url
906 * -- pPage: web page
907 * Output argv:
908 * --
909 * Return:
910 * Function Description: save raw page data as ISAM file
911 **************************************************************************************/
912 void CCrawl::SaveIsamRawData(CUrl *pUrl, CPage *pPage) {
913 if (!pUrl || !pPage) {
914 return;
915 }
916
917 file_arg arg;
918 arg.pUrl = pUrl;
919 arg.pPage = pPage;
920
921 pthread_mutex_lock(&mutexIsamFile);
922
923 m_isamFile.Write((void *) &arg);
924
925 pthread_mutex_unlock(&mutexIsamFile);
926 }
927
928 /**************************************************************************************
929 * Function name: SaveVisitedUrl
930 * Input argv:
931 * -- url: url
932 * Output argv:
933 * --
934 * Return:
935 * Function Description: save raw the Visited Url
936 **************************************************************************************/
937 void CCrawl::SaveVisitedUrl(string url) {
938 if (m_ofsVisitedUrlFile) {
939 pthread_mutex_lock(&mutexVisitedUrlFile);
940
941 m_ofsVisitedUrlFile << url << endl;
942
943 pthread_mutex_unlock(&mutexVisitedUrlFile);
944 }
945 }
946
947 void CCrawl::SaveUnreachHost(string host) {
948 CMD5 iMD5;
949 iMD5.GenerateMD5((unsigned char*) host.c_str(), host.size());
950 string strDigest = iMD5.ToString();
951 if (setUnreachHostMD5.find(strDigest) == setUnreachHostMD5.end()) {
952 pthread_mutex_lock(&mutexUnreachHost);
953
954 setUnreachHostMD5.insert(strDigest);
955 if (m_ofsUnreachHostFile) {
956 m_ofsUnreachHostFile << host << endl;
957 }
958
959 pthread_mutex_unlock(&mutexUnreachHost);
960 }
961 }
962
963 void CCrawl::SaveLink4SE(CPage *iPage) {
964 if (m_ofsLink4SEFile && iPage->m_nRefLink4SENum > 0) {
965 pthread_mutex_lock(&mutexLink4SEFile);
966
967 m_ofsLink4SEFile << "root_url: " << iPage->m_sUrl << endl;
968 m_ofsLink4SEFile << "charset: " << iPage->m_sCharset << endl;
969 m_ofsLink4SEFile << "number: " << iPage->m_nRefLink4SENum << endl;
970 m_ofsLink4SEFile << "link_anchortext: " << endl;
971
972 map<string, string>::iterator it4SE = iPage->m_mapLink4SE.begin();
973 for (; it4SE != iPage->m_mapLink4SE.end(); ++it4SE) {
974
975 m_ofsLink4SEFile << (*it4SE).first << '\t' << (*it4SE).second
976 << endl;
977 ;
978
979 }
980
981 pthread_mutex_unlock(&mutexLink4SEFile);
982 }
983 }
984
985 bool CCrawl::SaveLink4SE031121(void *arg) {
986 if (!arg || !m_ofsLink4SEFile)
987 return false;
988
989 //pthread_mutex_lock(&mutexLink4SEFile);
990
991 if (vsParsedLinks.size() == 0)
992 return false;
993
994 file_arg *pFile = (file_arg *) arg;
995
996 CUrl *iUrl = pFile->pUrl;
997 CPage *iPage = pFile->pPage;
998
999 char strDownloadTime[128];
1000 time_t tDate;
1001
1002 memset(strDownloadTime, 0, 128);
1003 time(&tDate);
1004 strftime(strDownloadTime, 128, "%a, %d %b %Y %H:%M:%S GMT", gmtime(&tDate));
1005
1006 string links;
1007 vector<string>::iterator it = vsParsedLinks.begin();
1008 for (; it != vsParsedLinks.end(); ++it) {
1009 links = links + *it + "\n";
1010 }
1011
1012 m_ofsLink4SEFile << "version: 1.0\n";
1013 if (iPage->m_sLocation.size() == 0) {
1014 m_ofsLink4SEFile << "url: " << iPage->m_sUrl;
1015 } else {
1016 m_ofsLink4SEFile << "url: " << iPage->m_sLocation;
1017 m_ofsLink4SEFile << "\norigin: " << iUrl->m_sUrl;
1018 }
1019
1020 m_ofsLink4SEFile << "\ndate: " << strDownloadTime;
1021
1022 if (mapCacheHostLookup.find(iUrl->m_sHost) == mapCacheHostLookup.end()) {
1023 m_ofsLink4SEFile << "\nip: " << iUrl->m_sHost;
1024 } else {
1025 m_ofsLink4SEFile << "\nip: "
1026 << (*(mapCacheHostLookup.find(iUrl->m_sHost))).second;
1027 }
1028
1029 m_ofsLink4SEFile << "\noutdegree: " << vsParsedLinks.size();
1030 m_ofsLink4SEFile << "\nlength: " << iPage->m_nLenHeader + links.size() + 1
1031 << "\n\n" << iPage->m_sHeader << "\n";
1032 m_ofsLink4SEFile << links;
1033 m_ofsLink4SEFile << endl;
1034
1035 vsParsedLinks.clear();
1036 //pthread_mutex_unlock(&mutexLink4SEFile);
1037
1038 return true;
1039 }
1040
1041 // not well
1042 void CCrawl::SaveLink4History(CPage *iPage) {//保存为历史网页存档准备的超链接信息
1043 if (m_ofsLink4HistoryFile && iPage->m_nRefLink4HistoryNum > 0) {
1044 pthread_mutex_lock(&mutexLink4HistoryFile);
1045
1046 //m_ofsLink4HistoryFile << "root_url: " << iPage->m_sUrl << endl;
1047 //m_ofsLink4HistoryFile << "charset: " << iPage->m_sCharset << endl;
1048 //m_ofsLink4HistoryFile << "number: " << iPage->m_nRefLink4HistoryNum << endl;
1049 //m_ofsLink4HistoryFile << "link: " << endl;
1050
1051 vector<string>::iterator it4History = iPage->m_vecLink4History.begin();
1052 for (; it4History != iPage->m_vecLink4History.end(); ++it4History) {
1053 string s = *it4History;
1054 m_ofsLink4HistoryFile << s << endl;
1055 }
1056
1057 pthread_mutex_unlock(&mutexLink4HistoryFile);
1058 }
1059 }
1060
1061 /**************************************************************************************
1062 * Function name: SaveVisitedUrlMd5
1063 * Input argv:
1064 * -- md5: page md5 value
1065 * Output argv:
1066 * --
1067 * Return:
1068 * Function Description: save the visited url Md5
1069 **************************************************************************************/
1070 void CCrawl::SaveVisitedUrlMD5(string md5) {
1071 if (m_ofsVisitedUrlMD5File) {
1072 m_ofsVisitedUrlMD5File << md5 << endl;
1073 }
1074 }
1075
1076 /**************************************************************************************
1077 * Function name: SaveVisitedPageMd5
1078 * Input argv:
1079 * -- md5: page md5 value
1080 * Output argv:
1081 * --
1082 * Return:
1083 * Function Description: save the visited url Md5
1084 **************************************************************************************/
1085 void CCrawl::SaveVisitedPageMD5(string md5) {
1086 if (m_ofsVisitedPageMD5File) {
1087 m_ofsVisitedPageMD5File << md5 << endl;
1088 }
1089 }
1090
1091 /**************************************************************************************
1092 * Function name: OpenFileForOutput
1093 * Input argv:
1094 * --
1095 * Output argv:
1096 * --
1097 * Return:
1098 * Function Description: Open the files for output
1099 **************************************************************************************/
1100 void CCrawl::OpenFilesForOutput() {
1101 // open isam file for output
1102 m_isamFile.Open(DATA_FILE_NAME, INDEX_FILE_NAME);
1103
1104 // open visited.url file for output
1105 m_ofsVisitedUrlFile.open(m_sOutputFileName.c_str(),
1106 ios::out | ios::app | ios::binary);
1107 if (!m_ofsVisitedUrlFile) {
1108 cerr << "cannot open " << VISITED_FILE << " for output\n" << endl;
1109 }
1110
1111 // open link4SE.url file for output
1112 m_ofsLink4SEFile.open(LINK4SE_FILE.c_str(),
1113 ios::out | ios::app | ios::binary);
1114 if (!m_ofsLink4SEFile) {
1115 cerr << "cannot open " << LINK4SE_FILE << " for output\n" << endl;
1116 }
1117
1118 // open link4History.url file for output
1119 m_ofsLink4HistoryFile.open(LINK4History_FILE.c_str(),
1120 ios::out | ios::app | ios::binary);
1121 if (!m_ofsLink4HistoryFile) {
1122 cerr << "cannot open " << LINK4History_FILE << " for output\n" << endl;
1123 }
1124
1125 // open unreach host file for output
1126 m_ofsUnreachHostFile.open(UNREACH_HOST_FILE.c_str(),
1127 ios::out | ios::app | ios::binary);
1128 if (!m_ofsUnreachHostFile) {
1129 cerr << "cannot open " << UNREACH_HOST_FILE << " for output\n" << endl;
1130 }
1131
1132 // open visited url md5 file for output
1133 m_ofsVisitedUrlMD5File.open(URL_MD5_FILE.c_str(),
1134 ios::out | ios::app | ios::binary);
1135 if (!m_ofsVisitedUrlMD5File) {
1136 cerr << "cannot open " << URL_MD5_FILE << " for output\n" << endl;
1137 }
1138
1139 // open visited page md5 file for output
1140 m_ofsVisitedPageMD5File.open(PAGE_MD5_FILE.c_str(),
1141 ios::out | ios::app | ios::binary);
1142 if (!m_ofsVisitedPageMD5File) {
1143 cerr << "cannot open " << PAGE_MD5_FILE << " for output\n" << endl;
1144 }
1145 }
1146
1147 /***************************************************************************************
1148 * Function name: DoCrawl
1149 * Input argv:
1150 * --
1151 * Output argv:
1152 * --
1153 * Return:
1154 * Function Description: the main function for crawl
1155 * Be careful:
1156 ***************************************************************************************/
1157 void CCrawl::DoCrawl() {//CCrawl类中的总控函数
1158 /* set the signal function */
1159 signal(SIGTERM, SigTerm);
1160 signal(SIGKILL, SigTerm);
1161 signal(SIGINT, SigTerm);
1162 signal(SIGPIPE, SIG_IGN);
1163 signal(SIGCHLD, SIG_IGN);
1164
1165 // output the begin time
1166 char strTime[128];
1167 time_t tDate;
1168
1169 memset(strTime, 0, 128);
1170 time(&tDate);
1171 strftime(strTime, 128, "%a, %d %b %Y %H:%M:%S GMT", gmtime(&tDate));
1172 cout << "\n\nBegin at: " << strTime << "\n\n";
1173
1174 // get the other info from file
1175 GetVisitedUrlMD5();
1176 GetVisitedPageMD5();
1177
1178 GetIpBlock();
1179
1180 GetUnreachHostMD5();
1181
1182 // open the seed url file
1183 ifstream ifsSeed(m_sInputFileName.c_str());
1184 if (!ifsSeed) {
1185 cerr << "Cannot open " << m_sInputFileName << " for input\n";
1186 return;
1187 }
1188
1189 // open the files for output
1190 OpenFilesForOutput();
1191
1192 // Create thread ID structures.
1193 pthread_t *tids = (pthread_t*) malloc(NUM_WORKERS * sizeof(pthread_t));
1194 if (tids == NULL) {
1195 cerr << "malloc error" << endl;
1196 }
1197
1198 for (unsigned int i = 0; i < NUM_WORKERS; i++) {
1199 if (pthread_create(&tids[i], NULL, start, this))
1200 cerr << "create threads error" << endl;
1201 }
1202
1203 string strUrl;
1204 CPage iCPage;
1205 while (getline(ifsSeed, strUrl)) {
1206 string::size_type idx;
1207
1208 if (strUrl[0] == '\0' || strUrl[0] == '#' || strUrl[0] == '\n') {
1209 continue;
1210 }
1211
1212 idx = strUrl.find('\t');
1213 if (idx != string::npos) {
1214 strUrl = strUrl.substr(0, idx);
1215 }
1216
1217 //idx = strUrl.find("http");
1218 idx = CStrFun::FindCase(strUrl, "http");
1219 if (idx == string::npos) {
1220 //continue;
1221 idx = strUrl.find('/');
1222 if (idx == string::npos) {
1223 strUrl = "http://" + strUrl + "/";
1224 } else {
1225 strUrl = "http://" + strUrl;
1226 }
1227 }
1228
1229 //if( strUrl.length() < 8 ) continue;
1230
1231 if (iCPage.IsFilterLink(strUrl))
1232 continue;
1233 AddUrl(strUrl.c_str());
1234 }
1235
1236 // Get the unvisited URL
1237 ifstream ifsUnvisitedUrl(UNVISITED_FILE.c_str());
1238 if (ifsUnvisitedUrl) {
1239 while (getline(ifsUnvisitedUrl, strUrl)) {
1240 string::size_type idx;
1241
1242 if (strUrl[0] == '\0' || strUrl[0] == '#' || strUrl[0] == '\n') {
1243 continue;
1244 }
1245
1246 idx = strUrl.find('\t');
1247 if (idx != string::npos) {
1248 strUrl = strUrl.substr(0, idx);
1249 }
1250
1251 // filter invalid urls
1252 if (iCPage.IsFilterLink(strUrl))
1253 continue;
1254
1255 AddUrl(strUrl.c_str());
1256 }
1257 } else {
1258 //cerr << "Cannot open " << UNVISITED_FILE << " for input\n";
1259 }
1260
1261 // sleep(30);
1262 b_fOver = true;
1263 cout << "finished to get all unvisited urls." << endl;
1264
1265 // Wait for the threads.
1266 for (unsigned int i = 0; i < NUM_WORKERS; ++i) {
1267 (void) pthread_join(tids[i], NULL);
1268 }
1269
1270 cout << "closed " << NUM_WORKERS << " threads." << endl;
1271
1272 SaveUnvisitedUrl();
1273 SaveReplicas("repli");
1274
1275 memset(strTime, 0, 128);
1276 time(&tDate);
1277 strftime(strTime, 128, "%a, %d %b %Y %H:%M:%S GMT", gmtime(&tDate));
1278 cout << "\n\nEnd at: " << strTime << "\n\n";
1279 }
1280
1281 /*****************************************************************
1282 ** Function name: AddUrl
1283 ** Input argv:
1284 ** --
1285 ** Output argv:
1286 ** --
1287 ** Return:
1288 ** Function Description: Add a parsed url into the collection
1289 ** Version: 1.0
1290 ** Be careful: An important function!!!
1291 *****************************************************************/
1292 void CCrawl::AddUrl(const char * url) {
1293 string strUrl = url;
1294 if (strUrl.empty() || strUrl.size() < 8) { //invalid url
1295 cout << "!so small!" << strUrl << endl;
1296 return;
1297 }
1298
1299 CPage iCPage;
1300 if (iCPage.NormalizeUrl(strUrl) == false) {
1301 // cout << "!normalize fail!" << strUrl << endl;
1302 return;
1303 }
1304
1305 CUrl iUrl;
1306
1307 // for ImgSE, comment the paragraph
1308 // if image/xxx url, store it to link4History.url
1309 // begin
1310 if (iUrl.IsImageUrl(strUrl)) {
1311 if (m_ofsLink4HistoryFile) {
1312 pthread_mutex_lock(&mutexLink4HistoryFile);
1313 m_ofsLink4HistoryFile << strUrl << endl;
1314 ;
1315 pthread_mutex_unlock(&mutexLink4HistoryFile);
1316 }
1317 return;
1318 }
1319 // end
1320
1321 if (iUrl.ParseUrlEx(strUrl) == false) {
1322 cout << "ParseUrlEx error in AddUrl(): " << strUrl << endl;
1323 return;
1324 }
1325
1326 // if it is an invalid host, discard it
1327 if (iUrl.IsValidHost(iUrl.m_sHost.c_str()) == false) {
1328 cout << "!invalid host: " << iUrl.m_sHost << endl;
1329 return;
1330 }
1331
1332 // filter foreign hosts
1333 if (iUrl.IsForeignHost(iUrl.m_sHost)) {
1334 cout << "!foreign hosts: " << iUrl.m_sHost << endl;
1335 return;
1336 }
1337
1338 // if it is a block ip, discard it
1339 // this work is left in the CreatSocket()
1340 // because the work of getting ip is inevitable in the CreatSocket function
1341 // and this work is expensive
1342 // if it is an unreach host, discard it
1343 // here we only deal with numbers-and-dots notations
1344 unsigned long inaddr = 0;
1345 char *ip = NULL;
1346
1347 inaddr = (unsigned long) inet_addr(iUrl.m_sHost.c_str());
1348 if (inaddr != INADDR_NONE) { // host is just ip
1349 //pthread_mutex_lock(&mutexMemory);
1350 ip = new char[iUrl.m_sHost.size() + 1];
1351 //pthread_mutex_unlock(&mutexMemory);
1352 memset(ip, 0, iUrl.m_sHost.size() + 1);
1353 memcpy(ip, iUrl.m_sHost.c_str(), iUrl.m_sHost.size());
1354
1355 if (!iUrl.IsValidIp(ip)) { // out of ip block
1356 //pthread_mutex_lock(&mutexMemory);
1357 delete[] ip;
1358 ip = NULL;
1359 //pthread_mutex_unlock(&mutexMemory);
1360 //cout << "!unreach hosts: " << iUrl.m_sHost << endl;
1361 return;
1362 }
1363 //pthread_mutex_lock(&mutexMemory);
1364 delete[] ip;
1365 ip = NULL;
1366 //pthread_mutex_unlock(&mutexMemory);
1367 }
1368
1369 CStrFun::Str2Lower(iUrl.m_sHost, iUrl.m_sHost.size());
1370 CMD5 iMD5;
1371 iMD5.GenerateMD5((unsigned char*) iUrl.m_sHost.c_str(),
1372 iUrl.m_sHost.size());
1373 string strDigest = iMD5.ToString();
1374 if (setUnreachHostMD5.find(strDigest) != setUnreachHostMD5.end()) {
1375 //cout << "!unreach host! " << iUrl.m_sHost << endl;
1376 return;
1377 }
1378
1379 // if crawled, discard it
1380 iMD5.GenerateMD5((unsigned char*) strUrl.c_str(), strUrl.size());
1381 strDigest = iMD5.ToString();
1382
1383 if (setVisitedUrlMD5.find(strDigest) != setVisitedUrlMD5.end()) {
1384 // cout << "!visited! " << strUrl << endl;
1385 return;
1386 }
1387
1388 // if already in the collection, discard it
1389 if (setUnvisitedUrlMD5.find(strDigest) != setUnvisitedUrlMD5.end()) {
1390 // cout << "!in collection! " << strUrl << endl;
1391 return;
1392 } else {
1393 pthread_mutex_lock(&mutexUnvisitedUrlMD5);
1394 setUnvisitedUrlMD5.insert(strDigest);
1395 pthread_mutex_unlock(&mutexUnvisitedUrlMD5);
1396 }
1397
1398 // add
1399 // make sure limited threads crawling on a site
1400 int cnt = 0;
1401 for (;;) {
1402 //if( mmapUrls.count(iUrl.m_sHost) < NUM_WORKERS_ON_A_SITE ){
1403
1404 if (1) {
1405 //pthread_mutex_lock(&mutexVisitedUrlMD5);
1406
1407 // if crawled, discard it :) double secure
1408 //if( setVisitedUrlMD5.find(strDigest) != setVisitedUrlMD5.end() ) {
1409 //cout << "!v! " << strUrl << endl;
1410 //pthread_mutex_unlock(&mutexVisitedUrlMD5);
1411 //return;
1412 //} else {
1413
1414 pthread_mutex_lock(&mutexVisitedUrlMD5);
1415 mmapUrls.insert(mvalType(iUrl.m_sHost, strUrl));
1416 pthread_mutex_unlock(&mutexVisitedUrlMD5);
1417 break;
1418 //}
1419 } else {
1420 cnt++;
1421 if (cnt % 100 == 0) {
1422 cout << "~";
1423 //cnt = 0;
1424 }
1425
1426 // If we have waiting so long, we may remove it
1427 if (cnt == 50000) {
1428 cout << "romove it!!!!!!!!!!!!!!!!!!!" << endl;
1429 break;
1430 }
1431 usleep(4000);
1432 }
1433
1434 }
1435
1436 }
