C# 模拟http请求网页数据 [网页爬虫]

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

using System;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using FTE.Framework.Log4NetService;
 
namespace Proxy.BllServices
{
    /// <summary>
    /// http访问类
    /// </summary>
    public class HttpHelper
    {
        /// <summary>
        /// 访问失败的统一返回字符
        /// </summary>
        public String ErrorReturn { get; private set; } = "HttpHelper access error!";
 
        /// <summary>
        /// 登录后保存的cookie
        /// </summary>
        private CookieContainer Cookie = new CookieContainer();
 
        /// <summary>
        /// http post 访问网页
        /// </summary>
        /// <param name="Url"></param>
        /// <param name="postDataStr"></param>
        /// <returns></returns>
        public string HttpPostString(string Url, string postDataStr)
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
                request.Method = "POST";
                request.ContentType = "application/x-www-form-urlencoded";
                request.ContentLength = Encoding.UTF8.GetByteCount(postDataStr);
                request.CookieContainer = Cookie;
                Stream myRequestStream = request.GetRequestStream();
                StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312"));
                myStreamWriter.Write(postDataStr);
                myStreamWriter.Close();
 
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                response.Cookies = Cookie.GetCookies(response.ResponseUri);
 
                Stream myResponseStream = response.GetResponseStream();
                StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                string retString = myStreamReader.ReadToEnd();
                myStreamReader.Close();
                myResponseStream.Close();
 
                return retString;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http post 网站出错", ex);
            }
 
            return ErrorReturn;
        }
 
        public string HttpGet(string Url, string postDataStr)
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url + (postDataStr == "" ? "" : "?") + postDataStr);
                request.Method = "GET";
                request.ContentType = "text/html;charset=UTF-8";
                request.CookieContainer = Cookie;
 
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream myResponseStream = response.GetResponseStream();
                StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                string retString = myStreamReader.ReadToEnd();
                myStreamReader.Close();
                myResponseStream.Close();
 
                return retString;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http get 网站出错", ex);
            }
 
            return ErrorReturn;
        }
 
        /// <summary>
        /// 使用form方式post数据[不包含文件]
        /// </summary>
        /// <param name="url"></param>
        /// <param name="stringDict"></param>
        /// <returns></returns>
        public string HttpPostForm(string url, NameValueCollection stringDict)
        {
            try
            {
                string responseContent;
                var memStream = new MemoryStream();
                var webRequest = (HttpWebRequest)WebRequest.Create(url);
                // 边界符  
                var boundary = "---------------" + DateTime.Now.Ticks.ToString("x");
                // 边界符  
                var beginBoundary = Encoding.ASCII.GetBytes("--" + boundary + "\r\n");
                // 最后的结束符  
                var endBoundary = Encoding.ASCII.GetBytes("--" + boundary + "--\r\n");
 
                // 设置属性  
                webRequest.CookieContainer = Cookie;
                webRequest.Method = "POST";
                webRequest.ContentType = "multipart/form-data; boundary=" + boundary;
                // 写入字符串的Key  
                var stringKeyHeader = "\r\n--" + boundary +
                                       "\r\nContent-Disposition: form-data; name=\"{0}\"" +
                                       "\r\n\r\n{1}\r\n";
 
                foreach (byte[] formitembytes in from string key in stringDict.Keys
                                                 select string.Format(stringKeyHeader, key, stringDict[key])
                                                     into formitem
                                                 select Encoding.UTF8.GetBytes(formitem))
                {
                    memStream.Write(formitembytes, 0, formitembytes.Length);
                }
 
                // 写入最后的结束边界符  
                memStream.Write(endBoundary, 0, endBoundary.Length);
 
                webRequest.ContentLength = memStream.Length;
 
                var requestStream = webRequest.GetRequestStream();
 
                memStream.Position = 0;
                var tempBuffer = new byte[memStream.Length];
                memStream.Read(tempBuffer, 0, tempBuffer.Length);
                memStream.Close();
 
                requestStream.Write(tempBuffer, 0, tempBuffer.Length);
                requestStream.Close();
 
                var httpWebResponse = (HttpWebResponse)webRequest.GetResponse();
 
                using (var httpStreamReader = new StreamReader(httpWebResponse.GetResponseStream(),
                                                                Encoding.GetEncoding("utf-8")))
                {
                    responseContent = httpStreamReader.ReadToEnd();
                }
 
                httpWebResponse.Close();
                webRequest.Abort();
 
                return responseContent;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http form post 网站出错", ex);
            }
 
            return ErrorReturn;
        }
    }
}

　　调用例子：

//1.
helper.HttpPostString("http://192.168.1.1/", "luci_username=root&luci_password=password");
 
//2.
NameValueCollection stringDict = new NameValueCollection();
stringDict.Add("token", token);
stringDict.Add("cbid.wireless.default_radio1.ssid", "everTestWifi");
helper.HttpPostData("http://192.168.1.1/cgi-bin/luci/admin/network/wireless/radio1.network2", stringDict);