换 代理 IP
和数据采集原理 差不多的。
http://www.haozs.net/proxyip/
/// <summary>
/// 下载文件
/// </summary>
/// <param name="PageUrl">网址 </param>
/// <param name="filename">保存文件路径 </param>
public void DownFile(string PageUrl, string filename)
{
if (!Directory.Exists(filename))
{
Directory.CreateDirectory(filename);
}
string path = PageUrl.Substring(PageUrl.LastIndexOf("/") + 1);
string dirname = filename + "\\" + path;
if (File.Exists(dirname))
{
return;
}
else
{
try
{
WebClient wc = new WebClient();
WebProxy wp = new WebProxy(al[proxy].ToString(), true);
wc.Proxy = wp;
wc.DownloadFile(PageUrl, dirname);
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
{
//文件未找到--跳出
//MessageBox.Show(ex.ToString());
return;
}
else
{
//换代理 IP
//MessageBox.Show(ex.ToString());
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
DownFile(PageUrl, filename);
}
}
}
}
==========
/// <summary>
/// 请求失败的时候,反复操作
/// </summary>
/// <param name="PageUrl"> </param>
/// <returns> </returns>
public string ToServer(string PageUrl)
{
string responseFromServer = "";
try
{
while (1 == 1)
{
WebRequest request = WebRequest.Create(PageUrl);
WebProxy wp = new WebProxy(al[proxy].ToString(), true);
request.Proxy = wp;
request.Timeout = 1000 * 45;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream dataStream = response.GetResponseStream();
StreamReader reader=null;
reader = new StreamReader(dataStream, System.Text.Encoding.Default);
responseFromServer = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
if (responseFromServer.Contains("refresh") ¦ ¦ responseFromServer == "")
{
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
//ToServer(PageUrl);
}
else
{
break;
}
}
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
{
responseFromServer = "";
}
else
{
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
ToServer(PageUrl);
}
}
return responseFromServer;
}
和数据采集原理 差不多的。
http://www.haozs.net/proxyip/
/// <summary>
/// 下载文件
/// </summary>
/// <param name="PageUrl">网址 </param>
/// <param name="filename">保存文件路径 </param>
public void DownFile(string PageUrl, string filename)
{
if (!Directory.Exists(filename))
{
Directory.CreateDirectory(filename);
}
string path = PageUrl.Substring(PageUrl.LastIndexOf("/") + 1);
string dirname = filename + "\\" + path;
if (File.Exists(dirname))
{
return;
}
else
{
try
{
WebClient wc = new WebClient();
WebProxy wp = new WebProxy(al[proxy].ToString(), true);
wc.Proxy = wp;
wc.DownloadFile(PageUrl, dirname);
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
{
//文件未找到--跳出
//MessageBox.Show(ex.ToString());
return;
}
else
{
//换代理 IP
//MessageBox.Show(ex.ToString());
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
DownFile(PageUrl, filename);
}
}
}
}
==========
/// <summary>
/// 请求失败的时候,反复操作
/// </summary>
/// <param name="PageUrl"> </param>
/// <returns> </returns>
public string ToServer(string PageUrl)
{
string responseFromServer = "";
try
{
while (1 == 1)
{
WebRequest request = WebRequest.Create(PageUrl);
WebProxy wp = new WebProxy(al[proxy].ToString(), true);
request.Proxy = wp;
request.Timeout = 1000 * 45;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream dataStream = response.GetResponseStream();
StreamReader reader=null;
reader = new StreamReader(dataStream, System.Text.Encoding.Default);
responseFromServer = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
if (responseFromServer.Contains("refresh") ¦ ¦ responseFromServer == "")
{
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
//ToServer(PageUrl);
}
else
{
break;
}
}
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
{
responseFromServer = "";
}
else
{
proxy++;
if (proxy >= al.Count)
{
al = ReadIPproxy("e:\\test.txt");//初始化代理 IP
}
ToServer(PageUrl);
}
}
return responseFromServer;
}