自己用C#写一个采集器、蜘蛛(zz)
效果图：
代码如下：
using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.Web;
using System.IO;
using System.Collections;
using System.Text.RegularExpressions;
namespace chinaz
{
class Program
{
static void Main(string[] args)
{
string cookie = null;
using (StreamReader sr = new StreamReader("cookie.txt"))
{
cookie = sr.ReadToEnd();
sr.Close();
}
//string tmp = SRWebClient.GetPage("http://bbs.chinaz.com/Members.html?page=1&sort=CreateDate&desc=true&keyword=", Encoding.UTF8, cookie);
int a = int.Parse(Console.ReadLine());
int b = int.Parse(Console.ReadLine());
string url = Console.ReadLine();
Hashtable hash = new Hashtable();
Encoding encoding = Encoding.GetEncoding(Console.ReadLine());
for (int i = a; i <= b; i++)
{
string html = SRWebClient.GetPage(string.Format(url, i), encoding, cookie);
//Console.WriteLine(html);
if (html != null && html.Length > 1000)
{
Match m = Regex.Match(html, @"\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*", RegexOptions.Compiled | RegexOptions.IgnoreCase);
while (m != null && m.Value != null && m.Value.Trim() != string.Empty)
{
if (!hash.Contains(m.Value))
{
Console.WriteLine(m.Value);
using (StreamWriter sw = new StreamWriter("mail.txt", true))
{
sw.WriteLine(m.Value);
sw.Close();
}
hash.Add(m.Value, string.Empty);
}
m = m.NextMatch();
}
}
}
Console.Write("完成");
Console.ReadLine();
}
}
public class SRWebClient
{
public CookieCollection cookie;
public SRWebClient()
{
cookie = null;
}
#region 从包含多个 Cookie 的字符串读取到 CookieCollection 集合中
private static void AddCookieWithCookieHead(ref CookieCollection cookieCol, string cookieHead, string defaultDomain)
{
if (cookieCol == null) cookieCol = new CookieCollection();
if (cookieHead == null) return;
string[] ary = cookieHead.Split(';');
for (int i = 0; i < ary.Length; i++)
{
Cookie ck = GetCookieFromString(ary[i].Trim(), defaultDomain);
if (ck != null)
{
cookieCol.Add(ck);
}
}
}
#endregion
#region 读取某一个 Cookie 字符串到 Cookie 变量中
private static Cookie GetCookieFromString(string cookieString, string defaultDomain)
{
string[] ary = cookieString.Split(',');
Hashtable hs = new Hashtable();
for (int i = 0; i < ary.Length; i++)
{
string s = ary[i].Trim();
int index = s.IndexOf("=");
if (index > 0)
{
hs.Add(s.Substring(0, index), s.Substring(index + 1));
}
}
Cookie ck = new Cookie();
foreach (object Key in hs.Keys)
{
if (Key.ToString() == "path") ck.Path = hs[Key].ToString();
else if (Key.ToString() == "expires")
{
//ck.Expires=DateTime.Parse(hs[Key].ToString();
}
else if (Key.ToString() == "domain") ck.Domain = hs[Key].ToString();
else
{
ck.Name = Key.ToString();
ck.Value = hs[Key].ToString();
}
}
if (ck.Name == "") return null;
if (ck.Domain == "") ck.Domain = defaultDomain;
return ck;
}
#endregion
/**/
/// <TgData>
/// <Alias>下载Web源代码</Alias>
/// </TgData>
public string DownloadHtml(string URL, bool CreateCookie)
{
try
{
HttpWebRequest request = HttpWebRequest.Create(URL) as HttpWebRequest;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
request.AllowAutoRedirect = false;
//request.MaximumAutomaticRedirections = 3;
request.Timeout = 20000;
HttpWebResponse res = (HttpWebResponse)request.GetResponse();
string r = "";
System.IO.StreamReader S1 = new System.IO.StreamReader(res.GetResponseStream(), System.Text.Encoding.Default);
try
{
r = S1.ReadToEnd();
if (CreateCookie)
cookie = res.Cookies;
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载Web错误", er.ToString());
}
finally
{
res.Close();
S1.Close();
}
return r;
}
catch
{
}
return string.Empty;
}
/**/
/// <TgData>
/// <Alias>下载文件</Alias>
/// </TgData>
public long DownloadFile(string FileURL, string FileSavePath, bool CreateCookie)
{
long Filelength = 0;
HttpWebRequest req = HttpWebRequest.Create(FileURL) as HttpWebRequest;
if (cookie != null)
{
req.CookieContainer = new CookieContainer();
req.CookieContainer.Add(cookie);
}
req.AllowAutoRedirect = true;
HttpWebResponse res = req.GetResponse() as HttpWebResponse;
if (CreateCookie)
cookie = res.Cookies;
System.IO.Stream stream = res.GetResponseStream();
try
{
Filelength = res.ContentLength;
byte[] b = new byte[512];
int nReadSize = 0;
nReadSize = stream.Read(b, 0, 512);
System.IO.FileStream fs = System.IO.File.Create(FileSavePath);
try
{
while (nReadSize > 0)
{
fs.Write(b, 0, nReadSize);
nReadSize = stream.Read(b, 0, 512);
}
}
finally
{
fs.Close();
}
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载文件错误", er.ToString());
}
finally
{
res.Close();
stream.Close();
}
return Filelength;
}
/**/
/// <TgData>
/// <Alias>提交数据</Alias>
/// </TgData>
public string Request(string RequestPageURL, RequestData Data, bool CreateCookie)
{
StreamReader reader = null;
HttpWebResponse response = null;
HttpWebRequest request = null;
try
{
string StrUrl = RequestPageURL;
request = HttpWebRequest.Create(StrUrl) as HttpWebRequest;
string postdata = Data.GetData();
request.Referer = RequestPageURL;
request.AllowAutoRedirect = false;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Timeout = 20000;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
Uri u = new Uri(StrUrl);
if (postdata.Length > 0) //包含要提交的数据 就使用Post方式
{
request.ContentType = "application/x-www-form-urlencoded"; //作为表单请求
request.Method = "POST"; //方式就是Post
//把提交的数据换成字节数组
Byte[] B = System.Text.Encoding.UTF8.GetBytes(postdata);
request.ContentLength = B.Length;
System.IO.Stream SW = request.GetRequestStream(); //开始提交数据
SW.Write(B, 0, B.Length);
SW.Close();
}
response = request.GetResponse() as HttpWebResponse;
if (CreateCookie)
//cookie = response.Cookies;
AddCookieWithCookieHead(ref cookie, response.Headers["Set-Cookie"], request.RequestUri.Host);
reader = new StreamReader(response.GetResponseStream(), Encoding.Default);
return reader.ReadToEnd();
}
catch (Exception ex)
{
string x = ex.StackTrace;
}
finally
{
if (response != null)
response.Close();
}
return string.Empty;
}
public bool PostDownload(RequestData Data, out string file)
{
file = null;
StreamReader reader = null;
HttpWebResponse response = null;
HttpWebRequest request = null;
try
{
string StrUrl = "http://www.imobile.com.cn/wapdiyringdownload.php";
request = HttpWebRequest.Create(StrUrl) as HttpWebRequest;
string postdata = Data.GetData();
request.Referer = StrUrl;
request.AllowAutoRedirect = false;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Timeout = 20000;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
Uri u = new Uri(StrUrl);
if (postdata.Length > 0) //包含要提交的数据 就使用Post方式
{
request.ContentType = "application/x-www-form-urlencoded"; //作为表单请求
request.Method = "POST"; //方式就是Post
//把提交的数据换成字节数组
Byte[] B = System.Text.Encoding.UTF8.GetBytes(postdata);
request.ContentLength = B.Length;
System.IO.Stream SW = request.GetRequestStream(); //开始提交数据
SW.Write(B, 0, B.Length);
SW.Close();
}
response = request.GetResponse() as HttpWebResponse;
string des = response.Headers["Content-Disposition"].Trim();
file = des.Substring(des.IndexOf("filename=") + 9);
file = new Random().Next(100).ToString() + "/" + file;
System.IO.Stream stream = response.GetResponseStream();
try
{
int Filelength = (int)response.ContentLength;
byte[] b = new byte[512];
int nReadSize = 0;
nReadSize = stream.Read(b, 0, 512);
System.IO.FileStream fs = System.IO.File.Create("f:/mobileMusic/" + file);
try
{
while (nReadSize > 0)
{
fs.Write(b, 0, nReadSize);
nReadSize = stream.Read(b, 0, 512);
}
}
finally
{
fs.Close();
}
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载文件错误", er.ToString());
}
finally
{
response.Close();
stream.Close();
}
}
catch (Exception ex)
{
string x = ex.StackTrace;
}
finally
{
if (response != null)
response.Close();
}
return true;
}
#region GetPage
/// <summary>
/// 获取源代码
/// </summary>
/// <param name="url"></param>
/// <param name="coding"></param>
/// <param name="TryCount"></param>
/// <returns></returns>
public static string GetPage(string url, Encoding encoding, int TryCount)
{
for (int i = 0; i < TryCount; i++)
{
string result = GetPage(url, encoding, null);
if (result != null && result != string.Empty)
return result;
}
return string.Empty;
}
/// <summary>
/// 获取源代码
/// </summary>
/// <param name="url"></param>
/// <param name="coding"></param>
/// <returns></returns>
public static string GetPage(string url, Encoding encoding, string cookie)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
StreamReader reader = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2;)";
request.Timeout = 20000;
request.AllowAutoRedirect = false;
if (cookie != null)
request.Headers["Cookie"] = cookie;
response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
{
reader = new StreamReader(response.GetResponseStream(), encoding);
string html = reader.ReadToEnd();
return html;
}
}
catch
{
}
finally
{
if (response != null)
{
response.Close();
response = null;
}
if (reader != null)
reader.Close();
if (request != null)
request = null;
}
return string.Empty;
}
#endregion
}
public class RequestData
{
Hashtable hash = new Hashtable();
public RequestData()
{
}
public string GetData()
{
string r = "";
foreach (string key in hash.Keys)
{
if (r.Length > 0) r += "&";
r += key + "=" + hash[key];
}
return r;
}
public void AddField(string Field, string Value)
{
hash[Field] = Value;
}
}
}
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.Web;
using System.IO;
using System.Collections;
using System.Text.RegularExpressions;
namespace chinaz
{
class Program
{
static void Main(string[] args)
{
string cookie = null;
using (StreamReader sr = new StreamReader("cookie.txt"))
{
cookie = sr.ReadToEnd();
sr.Close();
}
//string tmp = SRWebClient.GetPage("http://bbs.chinaz.com/Members.html?page=1&sort=CreateDate&desc=true&keyword=", Encoding.UTF8, cookie);
int a = int.Parse(Console.ReadLine());
int b = int.Parse(Console.ReadLine());
string url = Console.ReadLine();
Hashtable hash = new Hashtable();
Encoding encoding = Encoding.GetEncoding(Console.ReadLine());
for (int i = a; i <= b; i++)
{
string html = SRWebClient.GetPage(string.Format(url, i), encoding, cookie);
//Console.WriteLine(html);
if (html != null && html.Length > 1000)
{
Match m = Regex.Match(html, @"\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*", RegexOptions.Compiled | RegexOptions.IgnoreCase);
while (m != null && m.Value != null && m.Value.Trim() != string.Empty)
{
if (!hash.Contains(m.Value))
{
Console.WriteLine(m.Value);
using (StreamWriter sw = new StreamWriter("mail.txt", true))
{
sw.WriteLine(m.Value);
sw.Close();
}
hash.Add(m.Value, string.Empty);
}
m = m.NextMatch();
}
}
}
Console.Write("完成");
Console.ReadLine();
}
}
public class SRWebClient
{
public CookieCollection cookie;
public SRWebClient()
{
cookie = null;
}
#region 从包含多个 Cookie 的字符串读取到 CookieCollection 集合中
private static void AddCookieWithCookieHead(ref CookieCollection cookieCol, string cookieHead, string defaultDomain)
{
if (cookieCol == null) cookieCol = new CookieCollection();
if (cookieHead == null) return;
string[] ary = cookieHead.Split(';');
for (int i = 0; i < ary.Length; i++)
{
Cookie ck = GetCookieFromString(ary[i].Trim(), defaultDomain);
if (ck != null)
{
cookieCol.Add(ck);
}
}
}
#endregion
#region 读取某一个 Cookie 字符串到 Cookie 变量中
private static Cookie GetCookieFromString(string cookieString, string defaultDomain)
{
string[] ary = cookieString.Split(',');
Hashtable hs = new Hashtable();
for (int i = 0; i < ary.Length; i++)
{
string s = ary[i].Trim();
int index = s.IndexOf("=");
if (index > 0)
{
hs.Add(s.Substring(0, index), s.Substring(index + 1));
}
}
Cookie ck = new Cookie();
foreach (object Key in hs.Keys)
{
if (Key.ToString() == "path") ck.Path = hs[Key].ToString();
else if (Key.ToString() == "expires")
{
//ck.Expires=DateTime.Parse(hs[Key].ToString();
}
else if (Key.ToString() == "domain") ck.Domain = hs[Key].ToString();
else
{
ck.Name = Key.ToString();
ck.Value = hs[Key].ToString();
}
}
if (ck.Name == "") return null;
if (ck.Domain == "") ck.Domain = defaultDomain;
return ck;
}
#endregion
/**/
/// <TgData>
/// <Alias>下载Web源代码</Alias>
/// </TgData>
public string DownloadHtml(string URL, bool CreateCookie)
{
try
{
HttpWebRequest request = HttpWebRequest.Create(URL) as HttpWebRequest;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
request.AllowAutoRedirect = false;
//request.MaximumAutomaticRedirections = 3;
request.Timeout = 20000;
HttpWebResponse res = (HttpWebResponse)request.GetResponse();
string r = "";
System.IO.StreamReader S1 = new System.IO.StreamReader(res.GetResponseStream(), System.Text.Encoding.Default);
try
{
r = S1.ReadToEnd();
if (CreateCookie)
cookie = res.Cookies;
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载Web错误", er.ToString());
}
finally
{
res.Close();
S1.Close();
}
return r;
}
catch
{
}
return string.Empty;
}
/**/
/// <TgData>
/// <Alias>下载文件</Alias>
/// </TgData>
public long DownloadFile(string FileURL, string FileSavePath, bool CreateCookie)
{
long Filelength = 0;
HttpWebRequest req = HttpWebRequest.Create(FileURL) as HttpWebRequest;
if (cookie != null)
{
req.CookieContainer = new CookieContainer();
req.CookieContainer.Add(cookie);
}
req.AllowAutoRedirect = true;
HttpWebResponse res = req.GetResponse() as HttpWebResponse;
if (CreateCookie)
cookie = res.Cookies;
System.IO.Stream stream = res.GetResponseStream();
try
{
Filelength = res.ContentLength;
byte[] b = new byte[512];
int nReadSize = 0;
nReadSize = stream.Read(b, 0, 512);
System.IO.FileStream fs = System.IO.File.Create(FileSavePath);
try
{
while (nReadSize > 0)
{
fs.Write(b, 0, nReadSize);
nReadSize = stream.Read(b, 0, 512);
}
}
finally
{
fs.Close();
}
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载文件错误", er.ToString());
}
finally
{
res.Close();
stream.Close();
}
return Filelength;
}
/**/
/// <TgData>
/// <Alias>提交数据</Alias>
/// </TgData>
public string Request(string RequestPageURL, RequestData Data, bool CreateCookie)
{
StreamReader reader = null;
HttpWebResponse response = null;
HttpWebRequest request = null;
try
{
string StrUrl = RequestPageURL;
request = HttpWebRequest.Create(StrUrl) as HttpWebRequest;
string postdata = Data.GetData();
request.Referer = RequestPageURL;
request.AllowAutoRedirect = false;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Timeout = 20000;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
Uri u = new Uri(StrUrl);
if (postdata.Length > 0) //包含要提交的数据 就使用Post方式
{
request.ContentType = "application/x-www-form-urlencoded"; //作为表单请求
request.Method = "POST"; //方式就是Post
//把提交的数据换成字节数组
Byte[] B = System.Text.Encoding.UTF8.GetBytes(postdata);
request.ContentLength = B.Length;
System.IO.Stream SW = request.GetRequestStream(); //开始提交数据
SW.Write(B, 0, B.Length);
SW.Close();
}
response = request.GetResponse() as HttpWebResponse;
if (CreateCookie)
//cookie = response.Cookies;
AddCookieWithCookieHead(ref cookie, response.Headers["Set-Cookie"], request.RequestUri.Host);
reader = new StreamReader(response.GetResponseStream(), Encoding.Default);
return reader.ReadToEnd();
}
catch (Exception ex)
{
string x = ex.StackTrace;
}
finally
{
if (response != null)
response.Close();
}
return string.Empty;
}
public bool PostDownload(RequestData Data, out string file)
{
file = null;
StreamReader reader = null;
HttpWebResponse response = null;
HttpWebRequest request = null;
try
{
string StrUrl = "http://www.imobile.com.cn/wapdiyringdownload.php";
request = HttpWebRequest.Create(StrUrl) as HttpWebRequest;
string postdata = Data.GetData();
request.Referer = StrUrl;
request.AllowAutoRedirect = false;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Timeout = 20000;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
Uri u = new Uri(StrUrl);
if (postdata.Length > 0) //包含要提交的数据 就使用Post方式
{
request.ContentType = "application/x-www-form-urlencoded"; //作为表单请求
request.Method = "POST"; //方式就是Post
//把提交的数据换成字节数组
Byte[] B = System.Text.Encoding.UTF8.GetBytes(postdata);
request.ContentLength = B.Length;
System.IO.Stream SW = request.GetRequestStream(); //开始提交数据
SW.Write(B, 0, B.Length);
SW.Close();
}
response = request.GetResponse() as HttpWebResponse;
string des = response.Headers["Content-Disposition"].Trim();
file = des.Substring(des.IndexOf("filename=") + 9);
file = new Random().Next(100).ToString() + "/" + file;
System.IO.Stream stream = response.GetResponseStream();
try
{
int Filelength = (int)response.ContentLength;
byte[] b = new byte[512];
int nReadSize = 0;
nReadSize = stream.Read(b, 0, 512);
System.IO.FileStream fs = System.IO.File.Create("f:/mobileMusic/" + file);
try
{
while (nReadSize > 0)
{
fs.Write(b, 0, nReadSize);
nReadSize = stream.Read(b, 0, 512);
}
}
finally
{
fs.Close();
}
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载文件错误", er.ToString());
}
finally
{
response.Close();
stream.Close();
}
}
catch (Exception ex)
{
string x = ex.StackTrace;
}
finally
{
if (response != null)
response.Close();
}
return true;
}
#region GetPage
/// <summary>
/// 获取源代码
/// </summary>
/// <param name="url"></param>
/// <param name="coding"></param>
/// <param name="TryCount"></param>
/// <returns></returns>
public static string GetPage(string url, Encoding encoding, int TryCount)
{
for (int i = 0; i < TryCount; i++)
{
string result = GetPage(url, encoding, null);
if (result != null && result != string.Empty)
return result;
}
return string.Empty;
}
/// <summary>
/// 获取源代码
/// </summary>
/// <param name="url"></param>
/// <param name="coding"></param>
/// <returns></returns>
public static string GetPage(string url, Encoding encoding, string cookie)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
StreamReader reader = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2;)";
request.Timeout = 20000;
request.AllowAutoRedirect = false;
if (cookie != null)
request.Headers["Cookie"] = cookie;
response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
{
reader = new StreamReader(response.GetResponseStream(), encoding);
string html = reader.ReadToEnd();
return html;
}
}
catch
{
}
finally
{
if (response != null)
{
response.Close();
response = null;
}
if (reader != null)
reader.Close();
if (request != null)
request = null;
}
return string.Empty;
}
#endregion
}
public class RequestData
{
Hashtable hash = new Hashtable();
public RequestData()
{
}
public string GetData()
{
string r = "";
foreach (string key in hash.Keys)
{
if (r.Length > 0) r += "&";
r += key + "=" + hash[key];
}
return r;
}
public void AddField(string Field, string Value)
{
hash[Field] = Value;
}
}
}
