爬虫类与界面的交互

把一个C#控制台显示的爬虫程序，改写成界面的爬虫，爬取的网页地址显示到listBox：

一、爬虫类

　　1.爬虫类与界面建立联系，即在爬虫类增加一个Form类对象字段，爬虫类构造函数初始化Form类对象；

　　2.爬取方法，使用Form类对象调用Form类显示方法，在控制台的显示改成在listBox中显示；

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net;
using System.Collections;
using System.Threading;
using System.Text.RegularExpressions;
using System.IO;

namespace Crawler
{
    class Crawler
    {
        //字段
        private WebClient webClient = new WebClient();
        public Hashtable urls = new Hashtable();
        private int count = 0;
        private Form1 f;
               
        //构造
        public Crawler(Form1 f1)
        {
            f = f1;
        }
        //1.方法Crawl
        public void Crawl()
        {
            string disp = "开始爬行了....";
            f.addUrl(disp);
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys) //找到一个还没有下载过的链接
                {
                    if ((bool)urls[url]) continue; //已经下载过的，不再下载
                    current = url;
                    
                }
                if (current == null || count > 10) break;

                disp = "爬行" + current + "页面！";
                f.addUrl(disp);

                string html = DownLoad(current); //下载

                urls[current] = true;
                count++;

                Parse(html); //解析，并加入新的链接
            }
            disp = "爬行结束";
            f.addUrl(disp);
        }

        //2.方法DownLoad
        public string DownLoad(string url)
        {
            try
            {
                HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
                req.Timeout = 30000;
                HttpWebResponse response = (HttpWebResponse)req.GetResponse();
                byte[] buffer = ReadInstreamIntoMemory(response.GetResponseStream());
                string fileName = count.ToString();
                FileStream fs = new FileStream(fileName, FileMode.OpenOrCreate);
                fs.Write(buffer, 0, buffer.Length);
                fs.Close();
                string html = Encoding.UTF8.GetString(buffer);
                return html;
            }
            catch
            {
            }
            return "";
        }

        //3.方法 Parse
        public void Parse(string html)
        {
            string strRef = @"(href|HREF|src|SRC)[ ]*=[ ]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\'', '#', ' ', '>');
                if (strRef.Length == 0) continue;

                if (urls[strRef] == null) urls[strRef] = false;
            }
        }

        //4.方法 ReadInstreamIntoMemory
        private static byte[] ReadInstreamIntoMemory(Stream stream)
        {
            int bufferSize = 16384;
            byte[] buffer = new byte[bufferSize];
            MemoryStream ms = new MemoryStream();
            while (true)
            {
                int numBytesRead = stream.Read(buffer, 0, bufferSize);
                if (numBytesRead <= 0) break;
                ms.Write(buffer, 0, numBytesRead);
            }
            return ms.ToArray();
        }
    }
}

二、窗体类

　　button控件Click事件代码下创建爬虫类对象，传递窗体对象，调用爬虫方法。

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Net;
using System.Collections;
using System.Threading;
using System.Text.RegularExpressions;
using System.IO;

namespace Crawler
{
    public partial class Form1 : Form
    {       
        public Form1()
        {
            InitializeComponent();
        }

        private void button1_Click(object sender, EventArgs e)
        {
            Crawler myCrawler = new Crawler(this);
       
            string startUrl = null;
            if (textBox1.Text == "") return; 
                
            startUrl = textBox1.Text;   

            myCrawler.urls.Add(startUrl, false); //加入初始页面
            myCrawler.Crawl();    
        }   

        private void button2_Click(object sender, EventArgs e)
        {
            listBox1.Items.Clear();
            textBox1.Text = "";
            textBox1.Focus();
        }

        //输出
        public void addUrl(string disp)
        {
            listBox1.Items.Add(disp);         
            //选最后一条，到底部，然后取消
            listBox1.SelectedIndex = listBox1.Items.Count - 1;
            listBox1.SelectedIndex = -1;
        }
    }
}