using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Collections;
using System.Text.RegularExpressions;
namespace Net
{
public class HttpUtil
{
private string accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-shockwave-flash, application/xaml+xml, application/vnd.ms-xpsdocument, application/x-ms-xbap, application/x-ms-application, */*";
private string userAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
private HttpMethod method = HttpMethod.GET;
private HttpContentType contentType = HttpContentType.Application_x_www_form_urlencoded;
private bool keepAlive = true;
private bool allowAutoRedirect = true;
private bool expect100Continue = false;
private string url = string.Empty;
private string referer = string.Empty;
private string cookieHeader = string.Empty;
private readonly List<HttpPostData> postingData = new List<HttpPostData>();
private int fileCount = 0;
private bool addWWWToUrl = false;
private int timeout = 100000;
private bool hasMethodSet = false;
public string RequestEncoding { get; set; }
public string boundary { get; set; }
public string MultipartFormData { get; set; }
private Hashtable header = new Hashtable();
/// <summary>
/// 添加http标头
/// </summary>
/// <param name="key"></param>
/// <param name="value"></param>
public void AddRequestHeader(string key, string value)
{
header.Add(key, value);
}
/// <summary>
/// 构造新的HttpUtil实例
/// </summary>
/// <param name="url">请求的 Internet 资源的统一资源标识符</param>
public HttpUtil(string url)
: this(url, string.Empty, string.Empty)
{ }
/// <summary>
/// 构造新的HttpUtil实例
/// </summary>
/// <param name="url">请求的 Internet 资源的统一资源标识符</param>
/// <param name="referer">Referer HTTP 标头的值</param>
public HttpUtil(string url, string referer)
: this(url, referer, string.Empty)
{ }
/// <summary>
/// 构造新的HttpUtil实例
/// </summary>
/// <param name="url">请求的 Internet 资源的统一资源标识符</param>
/// <param name="referer">Referer HTTP 标头的值</param>
/// <param name="cookieHeader">Cookie HTTP 标头的值</param>
public HttpUtil(string url, string referer, string cookieHeader)
{
this.url = url;
this.referer = referer;
this.cookieHeader = cookieHeader;
}
/// <summary>
/// 获取或设置 Accept HTTP 标头的值
/// </summary>
public string Accept
{
get { return accept; }
set { accept = value; }
}
/// <summary>
/// 获取或设置 User-agent HTTP 标头的值
/// </summary>
public string UserAgent
{
get { return userAgent; }
set { userAgent = value; }
}
/// <summary>
/// 获取或设置请求的方法(Get请求还是Post请求)
/// </summary>
public HttpMethod Method
{
get { return method; }
set
{
method = value;
hasMethodSet = true;
}
}
/// <summary>
/// 获取或设置 Content-type HTTP 标头的值
/// </summary>
public HttpContentType ContentType
{
get { return contentType; }
set { contentType = value; }
}
/// <summary>
/// 获取或设置一个值，该值指示是否与 Internet 资源建立持久性连接
/// </summary>
public bool KeepAlive
{
get { return keepAlive; }
set { keepAlive = value; }
}
/// <summary>
/// 获取或设置一个值，该值指示请求是否应跟随重定向响应
/// </summary>
public bool AllowAutoRedirect
{
get { return allowAutoRedirect; }
set { allowAutoRedirect = value; }
}
/// <summary>
/// 获取或设置是否使用 100-Continue 行为
/// </summary>
public bool Expect100Continue
{
get { return expect100Continue; }
set { expect100Continue = value; }
}
/// <summary>
/// 获取或设置请求超时前等待的毫秒数。
/// </summary>
public int TimeOut
{
get { return timeout; }
set { timeout = value; }
}
/// <summary>
/// 获取或设置实际响应请求的 Internet 资源的统一资源标识符
/// </summary>
public string Url
{
get { return url; }
set { url = value; }
}
/// <summary>
/// 获取或设置 Referer HTTP 标头的值
/// </summary>
public string Referer
{
get { return referer; }
set { referer = value; }
}
/// <summary>
/// 获取或设置 Cookie HTTP 标头的值
/// </summary>
public string CookieHeader
{
get { return cookieHeader; }
set { cookieHeader = value; }
}
/// <summary>
/// 获取或设置Url是否补上www
/// </summary>
public bool AddWWWToUrl
{
get { return addWWWToUrl; }
set { addWWWToUrl = value; }
}
/// <summary>
/// 要上传的文件.如果不为空则自动转为Post请求
/// </summary>
public List<HttpPostData> Files
{
get
{
List<HttpPostData> files = new List<HttpPostData>();
foreach (HttpPostData file in postingData)
{
if (file.IsFile)
files.Add(file);
}
return files;
}
}
/// <summary>
/// 要发送的Form表单信息
/// </summary>
public List<HttpPostData> PostingData
{
get
{
List<HttpPostData> data = new List<HttpPostData>();
foreach (HttpPostData dt in postingData)
{
if (!dt.IsFile)
data.Add(dt);
}
return data;
}
}
/// <summary>
/// 在请求中添加要上传的文件
/// </summary>
/// <param name="fieldName">文件字段的名称</param>
/// <param name="fileName">要上传的文件完整路径</param>
public void AttachFile(string fieldName, string fileName)
{
HttpPostData file = new HttpPostData(fieldName, fileName, true);
postingData.Add(file);
fileCount++;
}
/// <summary>
/// 在请求中添加要上传的数据
/// </summary>
/// <param name="fieldName">普通字段的名称</param>
/// <param name="fieldValue">普通字段的值</param>
public void AddPostData(string fieldName, string fieldValue)
{
HttpPostData data = new HttpPostData(fieldName, fieldValue, false);
postingData.Add(data);
}
public void AddPostData(string data)
{
AddPostData(string.Empty, data);
}
public long ContentLength { get; set; }
private HttpWebRequest CreateRequest()
{
string newLine = "\r\n";
string boundary = "---------------------------7d81961530944";
if (!hasMethodSet)
{
if (postingData.Count > 0)
{
method = HttpMethod.POST;
}
else
{
method = HttpMethod.GET;
}
}
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.Timeout = timeout;
req.Method = method.ToString();
req.KeepAlive = keepAlive;
req.AllowAutoRedirect = allowAutoRedirect;
req.Accept = accept;
req.UserAgent = userAgent;
req.Referer = referer;
req.ServicePoint.Expect100Continue = expect100Continue;
req.CookieContainer = new CookieContainer();
foreach (var s in header.Keys)
{
req.Headers.Add(s.ToString(), header[s].ToString());
}
if (cookieHeader.Length > 0)
{
cookieHeader = cookieHeader.Replace("$", "");
string cookievalue = cookieHeader.Replace("; ", ",");
req.CookieContainer.SetCookies(new Uri("http://" + req.RequestUri.Host), cookievalue);
}
if (method == HttpMethod.POST || method == HttpMethod.PUT)
{
if (fileCount > 0)
contentType = HttpContentType.Multipart_form_data;
if (contentType == HttpContentType.Application_x_www_form_urlencoded)
{
req.ContentType = "application/x-www-form-urlencoded";
}
else if (contentType == HttpContentType.Multipart_form_data)
{
req.ContentType = "multipart/form-data; boundary=" + boundary;
}
else if (contentType == HttpContentType.Text_xml)
{
req.ContentType = "text/xml";
}
else if (contentType == HttpContentType.Application_xml)
{
req.ContentType = "application/xml";
}
else if (contentType == HttpContentType.application_json)
{
req.ContentType = "multipart/form-data; boundary=qeguwjqpcikhukevtqcuivtjcuvbbyfn";
}
Stream stream = req.GetRequestStream();
StreamWriter streamWriter = null;
if (RequestEncoding == null)
{
streamWriter = new StreamWriter(stream, Encoding.GetEncoding(437));
}
else
{
streamWriter = new StreamWriter(stream, Encoding.GetEncoding(RequestEncoding));
}
if (contentType != HttpContentType.Multipart_form_data)
{
StringBuilder sb = new StringBuilder();
foreach (HttpPostData data in postingData)
{
if (data.FieldName.Length > 0)
{
sb.AppendFormat("{0}={1}&", data.FieldName, data.FieldValue);
}
else
{
//适用于Amazon上传Inventory
sb.AppendFormat("{0}&", data.FieldValue);
}
}
if (sb.Length > 0)
{
sb.Length--;
streamWriter.Write(sb.ToString());
}
}
else
{
foreach (HttpPostData data in postingData)
{
if (!data.IsFile)
{
streamWriter.Write("--" + boundary + newLine);
streamWriter.Write("Content-Disposition: form-data; name=\"{0}\"{1}{1}", data.FieldName, newLine);
streamWriter.Write(data.FieldValue + newLine);
}
else
{
if (!string.IsNullOrEmpty(data.FieldValue) && File.Exists(data.FieldValue))
{
string extension = (new FileInfo(data.FieldValue)).Extension;
if (extension.ToLower() != "gif")
extension = "pjpeg";
streamWriter.Write("--" + boundary + newLine);
streamWriter.Write(
"Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"{2}",
data.FieldName,
data.FieldValue,
newLine
);
streamWriter.Write("Content-Type: image/" + extension + newLine + newLine);
streamWriter.Flush();
FileStream fs = null;
try
{
fs = new FileStream(data.FieldValue, FileMode.Open, FileAccess.Read, FileShare.Read);
byte[] bytes = new byte[1024];
while (fs.CanRead)
{
int read = fs.Read(bytes, 0, 1024);
stream.Write(bytes, 0, read);
if (read == 0)
{
break;
}
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
fs.Close();
}
stream.Flush();
streamWriter.Write(newLine);
}
else
{
streamWriter.Write("--" + boundary + newLine);
streamWriter.Write(
"Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"{2}",
data.FieldName,
"",
newLine
);
streamWriter.Write("Content-Type: application/octet-stream" + newLine + newLine + newLine);
}
}
}
streamWriter.Write("--" + boundary + "--" + newLine);
}
streamWriter.Close();
stream.Close();
}
return req;
}
private HttpWebRequest CreateRequestData()
{
string newLine = "\r\n";
if (string.IsNullOrEmpty(this.boundary))
{
boundary = "----WebKitFormBoundaryrCnAdKfNQAfXw6iY";
}
if (!hasMethodSet)
{
if (postingData.Count > 0)
{
method = HttpMethod.POST;
}
else
{
method = HttpMethod.GET;
}
}
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.Timeout = timeout;
req.Method = method.ToString();
req.KeepAlive = keepAlive;
req.AllowAutoRedirect = allowAutoRedirect;
req.Accept = accept;
req.UserAgent = userAgent;
req.Referer = referer;
req.ServicePoint.Expect100Continue = expect100Continue;
req.CookieContainer = new CookieContainer();
foreach (var s in header.Keys)
{
req.Headers.Add(s.ToString(), header[s].ToString());
}
if (cookieHeader.Length > 0)
{
cookieHeader = cookieHeader.Replace("$", "");
string cookievalue = cookieHeader.Replace("; ", ",");
req.CookieContainer.SetCookies(new Uri("http://" + req.RequestUri.Host), cookievalue);
}
if (method == HttpMethod.POST || method == HttpMethod.PUT)
{
if (fileCount > 0)
contentType = HttpContentType.Multipart_form_data;
if (contentType == HttpContentType.Application_x_www_form_urlencoded)
{
req.ContentType = "application/x-www-form-urlencoded";
}
else if (contentType == HttpContentType.Multipart_form_data)
{
req.ContentType = "multipart/form-data; boundary=" + boundary;
}
else if (contentType == HttpContentType.Text_xml)
{
req.ContentType = "text/xml";
}
else if (contentType == HttpContentType.Application_xml)
{
req.ContentType = "application/xml";
}
else if (contentType == HttpContentType.application_json)
{
req.ContentType = "multipart/form-data; boundary=qeguwjqpcikhukevtqcuivtjcuvbbyfn";
}
Stream stream = req.GetRequestStream();
StreamWriter streamWriter = null;
if (RequestEncoding == null)
{
streamWriter = new StreamWriter(stream, Encoding.GetEncoding(437));
}
else
{
streamWriter = new StreamWriter(stream, Encoding.GetEncoding(RequestEncoding));
}
if (contentType != HttpContentType.Multipart_form_data)
{
StringBuilder sb = new StringBuilder();
foreach (HttpPostData data in postingData)
{
if (data.FieldName.Length > 0)
{
sb.AppendFormat("{0}={1}&", data.FieldName, data.FieldValue);
}
else
{
//适用于Amazon上传Inventory
sb.AppendFormat("{0}&", data.FieldValue);
}
}
if (sb.Length > 0)
{
sb.Length--;
streamWriter.Write(sb.ToString());
}
}
else
{
foreach (HttpPostData data in postingData)
{
if (!data.IsFile)
{
streamWriter.Write("--" + boundary + newLine);
streamWriter.Write("Content-Disposition: form-data; name=\"{0}\"{1}{1}", data.FieldName, newLine);
streamWriter.Write(data.FieldValue + newLine);
}
else
{
if (!string.IsNullOrEmpty(data.FieldValue) && File.Exists(data.FieldValue))
{
streamWriter.Write(MultipartFormData);
streamWriter.Flush();
FileStream fs = null;
try
{
fs = new FileStream(data.FieldValue, FileMode.Open, FileAccess.Read, FileShare.Read);
byte[] bytes = new byte[1024];
while (fs.CanRead)
{
int read = fs.Read(bytes, 0, 1024);
stream.Write(bytes, 0, read);
if (read == 0)
{
break;
}
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
fs.Close();
}
stream.Flush();
streamWriter.Write(newLine);
}
else
{
streamWriter.Write("--" + boundary + newLine);
streamWriter.Write(
"Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"{2}",
data.FieldName,
"",
newLine
);
streamWriter.Write("Content-Type: application/octet-stream" + newLine + newLine + newLine);
}
}
}
streamWriter.Write("--" + boundary + "--" + newLine);
}
streamWriter.Close();
stream.Close();
}
return req;
}
/// <summary>
/// 清空Post的数据,并把Method设置为Get
/// 在发出一个包含上述信息的请求后,必须调用此方法或手工设置相应属性以使下一次请求不会受到影响
/// </summary>
private void Reset()
{
method = HttpMethod.GET;
contentType = HttpContentType.Application_x_www_form_urlencoded;
keepAlive = true;
allowAutoRedirect = true;
expect100Continue = false;
addWWWToUrl = false;
fileCount = 0;
timeout = 150000;
postingData.Clear();
header.Clear();
hasMethodSet = false;
}
public HttpWebResponse GetResponse()
{
return GetResponse(false);
}
/// <summary>
/// 发出一次新的请求,并返回获得的回应
/// </summary>
/// <returns>相应的HttpWebResponse</returns>
public HttpWebResponse GetResponse(bool specialCookie)
{
ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
HttpWebRequest req = null;
if (!string.IsNullOrEmpty(this.MultipartFormData))
{
req = CreateRequestData();
}
else
{
req = CreateRequest();
}
HttpWebResponse rep;
try
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)3072;
rep = (HttpWebResponse)req.GetResponse();
}
catch (WebException ex)
{
rep = (HttpWebResponse)ex.Response;
if (rep == null)
throw ex;
}
string myCookies = string.Empty;
if (!addWWWToUrl)
{
myCookies = req.CookieContainer.GetCookieHeader(req.RequestUri);
}
else
{
string reqUrl = req.RequestUri.OriginalString;
int pos = reqUrl.IndexOf("//");
myCookies = req.CookieContainer.GetCookieHeader(new Uri(reqUrl.Substring(0, pos + 2) + "www." + reqUrl.Substring(pos + 2)));
}
cookieHeader = myCookies;
if (specialCookie)
{
CookieContainer contain = req.CookieContainer;
myCookies = rep.Headers["Set-cookie"];
string myCookiesCopy = myCookies;
if (!string.IsNullOrEmpty(myCookies))
{
//Regex re = new Regex("([^;,]+)=([^;,]+);Domain=([^;,]+);Path=([^;,]+)", RegexOptions.IgnoreCase);
Regex re = new Regex("([^;,]+)=([^;,]+);", RegexOptions.IgnoreCase);
MatchCollection mc = re.Matches(myCookies);
myCookies = cookieHeader;
foreach (Match m in mc)
{
//name, value, path, domain
string name = m.Groups[1].Value.Trim();
string value = m.Groups[2].Value;
if (name.ToLower() != "domain" && name.ToLower() != "path" && name.ToLower() != "expires")
{
//用户的电脑时钟大于服务器的时钟造成设置cookie直接失效,而php在设置cookie失效时会对内容增加deleted这个值
//例如在facebook中会导致图片无法正确提交
if (value.ToLower() != "deleted")
{
//查找过期时间，由于.net自带的方法在处理“, ”上有问题，只好自己来写
Match mc2 = Regex.Match(myCookiesCopy, name + "[^,]+expires=([^;]+)");
if (mc2.Success)
{
string expires = string.Empty;
expires = mc2.Groups[1].Value;
Match mc3 = Regex.Match(expires, ",[^,]+=");
if (mc3.Success)
{
expires = expires.Substring(0, mc3.Index);
}
if (expires.Length > 0)
{
DateTime expireDate;
if (DateTime.TryParse(expires, out expireDate))
{
if (expireDate < DateTime.Now)
continue;
}
}
}
if (myCookies.IndexOf(name + "=", StringComparison.InvariantCultureIgnoreCase) > -1)
{
myCookies = Regex.Replace(myCookies, name + "=[^;]+", name + "=" + value, RegexOptions.IgnoreCase);
}
else
{
myCookies = myCookies + (myCookies.Length > 0 ? "; " + name + "=" + value : name + "=" + value);
}
}
}
}
cookieHeader = myCookies;
}
}
referer = url;
Reset();
return rep;
}
/// <summary>
/// 发出一次新的请求,并返回获得的回应
/// </summary>
/// <returns>请求页面的HTML代码</returns>
public string GetHTMLString(bool specialCookie)
{
HttpWebResponse rep = GetResponse(specialCookie);
Stream responseStream = rep.GetResponseStream();
StreamReader responseReader = new StreamReader(responseStream, GetEncodingFromResponse(rep));
string html = responseReader.ReadToEnd();
responseReader.Close();
rep.Close();
return html;
}
public string GetGzipHTMLString(bool specialCookie)
{
HttpWebResponse rep = GetResponse(specialCookie);
Stream responseStream = rep.GetResponseStream();
//StreamReader responseReader = new StreamReader(responseStream, GetEncodingFromResponse(rep));
//string html = responseReader.ReadToEnd();
//responseReader.Close();
string html = "";
using (var zipStream = new System.IO.Compression.GZipStream(responseStream, System.IO.Compression.CompressionMode.Decompress))
{
using (StreamReader sr = new System.IO.StreamReader(zipStream, Encoding.UTF8))
{
html = sr.ReadToEnd();
}
}
rep.Close();
return html;
}
public string GetHTMLString()
{
return GetHTMLString(false);
}
/// <summary>
/// 根据已有的HttpWebResponse返回获得的回应
/// </summary>
/// <param name="rep">已有的HttpWebResponse</param>
/// <returns>请求页面的HTML代码</returns>
public string GetHTMLString(HttpWebResponse rep)
{
Stream responseStream = rep.GetResponseStream();
StreamReader responseReader = new StreamReader(responseStream, GetEncodingFromResponse(rep));
string html = responseReader.ReadToEnd();
return html;
}
/// <summary>
/// 根据http应答的http头来判断编码
/// </summary>
/// <param name="rep">产生的HttpWebResponse</param>
/// <returns></returns>
private Encoding GetEncodingFromResponse(HttpWebResponse rep)
{
Encoding encode = Encoding.Default;
string characterSet = rep.CharacterSet;
if (!string.IsNullOrEmpty(characterSet))
{
if (characterSet.IndexOf("ISO-8859-1", StringComparison.InvariantCultureIgnoreCase) > -1)
{
characterSet = "gb2312";
}
else if (characterSet.IndexOf("UTF-8", StringComparison.InvariantCultureIgnoreCase) > -1)
{
characterSet = "UTF-8";
}
encode = Encoding.GetEncoding(characterSet);
}
return encode;
}
}
public enum HttpMethod
{
GET,
POST,
PUT,
DELETE,
PATCH
}
public enum HttpContentType
{
Application_x_www_form_urlencoded,
Multipart_form_data,
Text_xml, //适用于Amazon上传Inventory
Application_xml,
application_json
}
public class HttpPostData
{
private string fieldName;
private string fieldValue;
private bool isFile;
public string FieldName
{
get { return fieldName; }
set { fieldName = value; }
}
public string FieldValue
{
get { return fieldValue; }
set { fieldValue = value; }
}
public bool IsFile
{
get { return isFile; }
set { isFile = value; }
}
public HttpPostData(string fieldName, string fieldValue, bool isFile)
{
this.fieldName = fieldName;
this.fieldValue = fieldValue;
this.isFile = isFile;
}
public override string ToString()
{
return FieldName + " : " + FieldValue;
}
}
}
