C# WebClient Get获取网页内容

//不知道怎么删除，只好留着

 1. Get方式：

WebClient web = new WebClient();
var html = web.DownloadString(url);

2. Post方式

        /// <summary>
        /// 
        /// </summary>
        /// <param name="web"></param>
        /// <param name="url"></param>
        /// <param name="queryString">格式： paramname=value@name2=value2</param>
        /// <param name="clearHeads"></param>
        /// <returns></returns>
        public static string Post(this MyWebClient web, string url, string queryString, bool clearHeads=false)
        {
            string postString = queryString;// WebUtility.UrlEncode( queryString);//这里即为传递的参数，可以用工具抓包分析，也可以自己分析，主要是form里面每一个name都要加进来  
            byte[] postData = Encoding.UTF8.GetBytes(postString);//编码，尤其是汉字，事先要看下抓取网页的编码方式 
            web.RequestConentLength = postData.Length;
            if (clearHeads)
            {
                web.Headers.Clear();
                web.Headers.Add("Content-Type", "application/x-www-form-urlencoded");//采取POST方式必须加的header，如果改为GET方式的话就去掉这句话即可  
            }
            
            byte[] responseData = web.UploadData(url, "POST", postData);//得到返回字符流  
            string srcString = Encoding.UTF8.GetString(responseData);//解码  
            return srcString;
        }

3. Headers设置

                    web.Headers.Add(HttpRequestHeader.Accept, "*/*");
                    web.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
                    web.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
                    //web.Headers.Add(HttpRequestHeader.Connection, "keep-alive");
                    web.Headers.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                    web.Headers.Add(HttpRequestHeader.Host, "wenshu.court.gov.cn");
                    web.Headers.Add("Origin", "http://wenshu.court.gov.cn");
                    //web.Headers.Add("Proxy-Connection", "keep-alive"); 
                    web.Headers.Add(HttpRequestHeader.UserAgent, userAgent);
                    web.Headers.Add("X-Requested-With", "XMLHttpRequest");
                    web.Headers.Add(HttpRequestHeader.Referer, WebUtility.UrlEncode(Referer1));

4.Cookie、超时等高可用基类

    public class MyWebClient : WebClient
    { 
        public CookieContainer Cookies ;

        public MyWebClient(CookieContainer cookieContainer)
        {
            this.Cookies = cookieContainer;
        }
         
        public int TimeoutSeconds { get; set; } = 60;

        public WebRequest Request { get; set; }

        public int RequestConentLength;
         
        protected override WebRequest GetWebRequest(Uri address)
        { 
            HttpWebRequest request = base.GetWebRequest(address) as HttpWebRequest;
            
            if (request != null)
            {
                request.Method = "Post";
                request.CookieContainer = Cookies;
                request.Timeout = 1000 * TimeoutSeconds;
                request.ContentLength = RequestConentLength;
            }
 
            Request = request;
            return request;
        }

        public WebResponse Response { get; set; }

        protected override WebResponse GetWebResponse(WebRequest request)
        {
            this.Response = base.GetWebResponse(request);
            return this.Response;
        }

        public string GetCookieValue(string cookieName)
        {
            var cookies = this.Cookies.GetCookies(this.Request.RequestUri);
            var ck = cookies[cookieName];
            return ck?.Value;
        }
    }

特别注意，一个浏览器对于网站多次请求的情况，要创建多个WebClient对象，但是它们应该共用一个CookieContainer。 而当写爬虫，模拟多个浏览器会话时，又不应全部使用同样的CookieContainer对象，以避免会话冲突。

 

附赠一个C#调用JS脚本的代码：

        public string CallJs(string jsCall , string jsFunctions)
        {
            Type obj = Type.GetTypeFromProgID("ScriptControl");
            if (obj == null) return null;
            object ScriptControl = Activator.CreateInstance(obj);
            obj.InvokeMember("Language", BindingFlags.SetProperty, null, ScriptControl, new object[] { "JavaScript" });
            //string js = "function time(a, b, msg){ var sum = a + b; return new Date().getTime() + ': ' + msg + ' = ' + sum }";
            obj.InvokeMember("AddCode", BindingFlags.InvokeMethod, null, ScriptControl, new object[] { jsFunctions });

            //return obj.InvokeMember("Eval", BindingFlags.InvokeMethod, null, ScriptControl, new object[] { "time(3, 5, '3 + 5')" }).ToString();
            return obj.InvokeMember("Eval", BindingFlags.InvokeMethod, null, ScriptControl, new object[] { jsCall }).ToString();
        }

使用示例：

string js = "function jsfunction(parm){ return parm + "abc"; }";
string val = CallJs($"jsfunction('{csvar}')", js.ToString());