HttpWebRequest 下载网页Html代码 下载文件（Remote和FTP）Get方式

在.net中可以使用XmlHttp，WebClient,HttpWebRequest等方式下载网页html源码。

使用XmlHttp需要引用Microsoft.Xml，在使用HttpWebRequest时，如果网站使用了反爬虫技术时，则需要为此模拟一个浏览器的环境访问，才能返回相应的html源码，否则将会是空，如下所示：

例如：某电子商务网站中有站内搜索

查看源码或浏览器上的URL

这样就可以使用GET直接请求。

public class WebPageUtil
    {
        //部分网站做了反爬虫技术时，需要模拟浏览器进行返回才能获取到相应的数据，否则获取不了
        private static CookieContainer cookie = new CookieContainer();
        private static string contentType = "application/x-www-form-urlencoded;";
        private static string accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/x-silverlight, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-silverlight-2-b1, */*";
        private static string userAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";

        /// <summary>
        /// 返回请求的URL地址Tuple&lt;bool,string,string&gt; = 是否成功，网页源码，异常信息
        /// </summary>
        /// <param name="url"></param>
        /// <param name="keyword"></param>
        /// <param name="encoding"></param>
        /// <param name="newUrl"></param>
        /// <returns></returns>
        public static Tuple<bool, string, string> GetHtmlSourceCode(string url, string keyword, Encoding encoding, out string newUrl)
        {
            bool methodStatus = false;
            string pageHtml = "", exceptionInfo = "";
            Tuple<bool, string, string> executeResult = new Tuple<bool, string, string>(methodStatus, pageHtml, exceptionInfo);
            //编码
            newUrl = url + System.Uri.EscapeUriString(keyword);
            //不编码：网站如果支持就可以不编码
            //newUrl = url + keyword;

            WebResponse response = null;
            HttpWebRequest request = null;
            Stream responseStream = null;
            StreamReader reader = null;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(newUrl);
                request.UserAgent = userAgent;
                request.ContentType = contentType;
                request.CookieContainer = cookie;
                request.Accept = accept;
                request.Method = "GET";
                request.Timeout = 30 * 1000;
                //request.Host = "www.suning.com";
                //request.UserAgent = "User-Agent    Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1";

                response = request.GetResponse();
                responseStream = response.GetResponseStream();
                reader = new StreamReader(responseStream, encoding);
                pageHtml = reader.ReadToEnd();
                methodStatus = true;
            }
            catch (System.Net.WebException err)
            {
                exceptionInfo = err.Message;
            }
            catch (Exception err)
            {
                exceptionInfo = err.Message;
            }
            finally
            {
                if (reader != null) reader.Close();
                if (responseStream != null) responseStream.Close();
                if (response != null) response.Close();
                if (request != null) request = null;
            }

            return Tuple.Create<bool, string, string>(methodStatus, pageHtml, exceptionInfo);
        }

调用时，直接将url和关键词组合后以GET方式就可以获取。

下载方法使用指定url地址通过的URI从远程服务器下载数据到本地应用程序.
1. 获得远程服务器url地址；
2. 获得目标文件路径；
3. 使用WebRequest对象检查文件是否存在于服务器端 (导入命名空间System.Net的引用)；
4. HTTP：创建WebClient(System.Net,类似于上面提到的UploadFile方法)实例,   访问你DownloadData() 方法通过指定URI下载文件缓冲资源到本地路径。实际上,对于HTTP资源,使用"GET"方法.

   FTP：创建FtpWebRequest实例,通过使用WebRequestMethods.Ftp.DownloadFile方法，我们可以接受来自服务器的资源流，此方法使用"RETR"命令下载FTP资源；
5. DownloadData方法会返回下载资源的字节数组，我们只需要从这个下载文件缓冲使用FileStream(using System.IO)写一个字节段到本地服务器路径；
6. 最后关闭并释放FileStream资源。

可参阅：WebClient 以及WebClient.DownloadData。

RemoteDownload

public abstract class RemoteDownload
    {

        public string UrlString{get;set;}
        public string DestDir{get;set;}
        public RemoteDownload(string urlString, string destDir)
        {
            this.UrlString = urlString;
            this.DestDir = destDir;
        }

        ///<summary>
        ///从远程服务器下载文件
        ///</summary>
        public virtual bool DownloadFile()
        {
            return true;
        }
    }

    /// <summary>
    /// HttpRemoteDownload 类
    /// </summary>
    public class HttpRemoteDownload : RemoteDownload
    {
        public HttpRemoteDownload(string urlString, string descFilePath)
            : base(urlString, descFilePath)
        {

        }

        public override bool DownloadFile()
        {
            string fileName = System.IO.Path.GetFileName(this.UrlString);
            string descFilePath =
                System.IO.Path.Combine(this.DestDir, fileName);
            try
            {
                WebRequest myre = WebRequest.Create(this.UrlString);
            }
            catch(Exception ex)
            {
                throw new Exception("服务器上不存在对应文件", ex.InnerException);
            }
            try
            {
                byte[] fileData;
                using (WebClient client = new WebClient())
                {
                    fileData = client.DownloadData(this.UrlString);
                }
                using (FileStream fs =
                      new FileStream(descFilePath, FileMode.OpenOrCreate))
                {
                    fs.Write(fileData, 0, fileData.Length);
                }
                return true;
            }
            catch (Exception ex)
            {
                throw new Exception("下载失败", ex.InnerException);
            }
        }
    }

FTPDownload

/// <summary>
    /// FtpDownload 类
    /// </summary>
    public class FtpRemoteDownload : RemoteDownload
    {
        public FtpRemoteDownload(string urlString, string descFilePath)
            : base(urlString, descFilePath)
        {

        }

        public override bool DownloadFile()
        {
            FtpWebRequest reqFTP;

            string fileName = System.IO.Path.GetFileName(this.UrlString);
            string descFilePath =
                System.IO.Path.Combine(this.DestDir, fileName);

            try
            {

                reqFTP = (FtpWebRequest)FtpWebRequest.Create(this.UrlString);
                reqFTP.Method = WebRequestMethods.Ftp.DownloadFile;
                reqFTP.UseBinary = true;

                using (FileStream outputStream = new FileStream(descFilePath, FileMode.OpenOrCreate))
                {
                    using (FtpWebResponse response = (FtpWebResponse)reqFTP.GetResponse())
                    {
                        using (Stream ftpStream = response.GetResponseStream())
                        {
                            int bufferSize = 2048;
                            int readCount;
                            byte[] buffer = new byte[bufferSize];
                            readCount = ftpStream.Read(buffer, 0, bufferSize);
                            while (readCount > 0)
                            {
                                outputStream.Write(buffer, 0, readCount);
                                readCount = ftpStream.Read(buffer, 0, bufferSize);
                            }
                        }
                    }

                }
                return true;
            }

            catch (Exception ex)
            {
                throw new Exception("下载失败", ex.InnerException);
            }
        }