第一篇 C#模拟http请求抓取数据

很多人用python去抓数据，语法简单又方便。我这里用C#去抓数据，python后期自学下，看下能写出一些好文吗？

上一篇 asp.net webService添加头文件验证 

 

第一步 先把应用程序发布iis服务器上

第二步 用谷歌浏览器抓下网络请求

观察头文件分析后 确认是post提交的。

请求一般看这些参数 status code 200 说明是正常的没有做跳转的包，长度是根据表单提交的参数计算出来了，如果发现计算的长度和抓取的包的长度不对 有可能是传递表单参数传入有问题 需要排查，

还有content-type 这个有可能会是application/json类型，传递参数那就是json类型了。

2.1 请求

2.2 响应

注意红框就是我想要拿到响应的数据，但是多了很多无用的html标签，这些无用的标签需要替换

参考文献：

正则表达式，去除所有HTML标签 https://www.cnblogs.com/caok168/articles/2567117.html

第三步 编写相关代码

using MyAutomaticRefund.Common;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace MyAutomaticRefund
{
    public partial class Form1 : Form
    {
        static CookieContainer m_Cookie = null;

        public Form1()
        {
            InitializeComponent();
        }

        private void button1_Click(object sender, EventArgs e)
        {
            string result = CaptureData();

            string regexstr = @"<[^>]*>";    //去除所有的标签
            result = Regex.Replace(result, regexstr, string.Empty, RegexOptions.IgnoreCase);
            MessageBox.Show(result.Trim());
        }

        /// <summary>
        /// 抓取按钮事件
        /// </summary>
        /// <returns></returns>
        private string CaptureData()
        {

            string result = "";
            try
            {
                string Url = "http://192.168.21.195:8044/WebForm1";
                HttpWebRequest request = WebRequest.Create(Url) as HttpWebRequest;
                request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
                request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36";
                request.Method = "POST";
                request.KeepAlive = true;
                request.Referer = "http://192.168.21.195:8044/WebForm1";
                request.ContentType = "application/x-www-form-urlencoded";
                request.Headers.Add("Accept-Encoding", "gzip, deflate");
                request.Headers.Add("Accept-Language", "zh-CN,zh;q=0.9");
                //request.Headers.Add("X-Requested-With", "XMLHttpRequest");
                request.CookieContainer = m_Cookie;
                request.Host = "192.168.21.195:8044";

                HttpRequestClient s = new HttpRequestClient(true);


                Dictionary<string, string> dic = new Dictionary<string, string>();
                dic.Add("__VIEWSTATE", "BPyDUNYWaU0XKZc1m/+hb4U4rlITcqsuUMRxGp8elUR/ZgK5owxYH030QThFNas5u/roAT5kBYSSMaMYwRFEcw1COVEwFNapeyfnLq2PZ98=");
                dic.Add("__VIEWSTATEGENERATOR","B6E7D48B");
                dic.Add("__EVENTVALIDATION", "3d+F/uurjOe90j09vxq1PU5Tli3EVssuM/HOISX5tyLUP2HdR956+2PC8ezRt2kDXMr1BDkstuMTV11I0laFwL+adHjgxQYtfEObtF/NligPunXkuGz4LuJ6VYdtACGg");
                dic.Add("Button1", "Button");

                string formData = ConvertStrDic(dic);
                //string formData = "__VIEWSTATE=BPyDUNYWaU0XKZc1m%2F%2Bhb4U4rlITcqsuUMRxGp8elUR%2FZgK5owxYH030QThFNas5u%2FroAT5kBYSSMaMYwRFEcw1COVEwFNapeyfnLq2PZ98%3D&__VIEWSTATEGENERATOR=B6E7D48B&__EVENTVALIDATION=3d%2BF%2FuurjOe90j09vxq1PU5Tli3EVssuM%2FHOISX5tyLUP2HdR956%2B2PC8ezRt2kDXMr1BDkstuMTV11I0laFwL%2BadHjgxQYtfEObtF%2FNligPunXkuGz4LuJ6VYdtACGg&Button1=Button";
                byte[] data = Encoding.UTF8.GetBytes(formData);

                request.ContentLength = formData.Length;
                using (Stream reqStream = request.GetRequestStream())
                {
                    reqStream.Write(data, 0, data.Length);
                    reqStream.Close();
                }

                //响应  
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                int statusInt = response.StatusCode.GetHashCode();
                //响应成功  
                if (response.StatusCode.ToString().ToLower() == "ok")
                {
                    result = s.getResponseBody(response);
                    return result;
                }
                return result;
            }
            catch (Exception ex)
            {
                return result;
            }

        }

        string ConvertStrDic(Dictionary<string, string> dic)
        {
            StringBuilder formdata = new StringBuilder();
            foreach (KeyValuePair<string, string> item in dic)
            {
                formdata.Append(item.Key).Append("=").Append(System.Web.HttpUtility.UrlEncode(item.Value)).Append("&");//注意传递的参数需要编码
            }
            return formdata.ToString().TrimEnd('&');
        }


        public class formTable
        {
            public string __VIEWSTATE { get; set; }
            public string __VIEWSTATEGENERATOR { get; set; }
            public string __EVENTVALIDATION { get; set; }
            public string Button1 { get; set; }
        }
    }
}

运行结果

 

用python语言 怎么写

这里我用到的库是 urllib

用到了request ,urlopen等语法

#第二种思路
import urllib.parse
import urllib.request

#data ="__VIEWSTATE=BPyDUNYWaU0XKZc1m%2F%2Bhb4U4rlITcqsuUMRxGp8elUR%2FZgK5owxYH030QThFNas5u%2FroAT5kBYSSMaMYwRFEcw1COVEwFNapeyfnLq2PZ98%3D&__VIEWSTATEGENERATOR=B6E7D48B&__EVENTVALIDATION=3d%2BF%2FuurjOe90j09vxq1PU5Tli3EVssuM%2FHOISX5tyLUP2HdR956%2B2PC8ezRt2kDXMr1BDkstuMTV11I0laFwL%2BadHjgxQYtfEObtF%2FNligPunXkuGz4LuJ6VYdtACGg&Button1=Button"

str = {
        '__VIEWSTATE': 'i8VVE3gtBLKgjBpYFwMCruW86sOjv2lTpmzZF3mD3L/QIkX0Ode3Xc9MaMXFQnjiAK80xxkQ9rjTyjGrBWvbLwcxug4r9Akhmcxs/plCdYM=',
        '__VIEWSTATEGENERATOR':'B6E7D48B',
        '__EVENTVALIDATION':'2X6ageL+LlYeiTSgyQPd/FPAhPtg350MNiiKvURoS4xMsysX+0HyGjGN93yx7K27/NubbDHt2oTpWbFCv1dampTLrZkWvsf32lHlqJUliAsudoGhZ3kwM/XCxXSlvhNr',
        'Button1':'Button'
}

# headers={
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
# }
# str = {'__VIEWSTATE': 'i8VVE3gtBLKgjBpYFwMCruW86sOjv2lTpmzZF3mD3L/QIkX0Ode3Xc9MaMXFQnjiAK80xxkQ9rjTyjGrBWvbLwcxug4r9Akhmcxs/plCdYM=','__VIEWSTATEGENERATOR':'B6E7D48B','__EVENTVALIDATION':'2X6ageL+LlYeiTSgyQPd/FPAhPtg350MNiiKvURoS4xMsysX+0HyGjGN93yx7K27/NubbDHt2oTpWbFCv1dampTLrZkWvsf32lHlqJUliAsudoGhZ3kwM/XCxXSlvhNr','Button1':'Button'}

data = bytes(urllib.parse.urlencode(str),encoding='utf-8')
url = "http://192.168.21.195:8044/WebForm1"
# request = urllib.request.Request(url=url,data=data,headers=headers,origin_req_host='192.168.21.195:8044',unverifiable=True,method='POST')
#第一种写法
# request = urllib.request.Request(url=url,data=data)
# response = urllib.request.urlopen(request)
# print(response.read().decode('utf-8'))
#第二种写法
response = urllib.request.urlopen(url,data=data)
print(response.read().decode('utf-8'))
# print(response.read()) 不设置编码格式是不能拿到想要的数据的

太乱了，整理一下把调试代码拿掉

#第二种思路
import urllib.parse
import urllib.request

str = {
        '__VIEWSTATE': 'i8VVE3gtBLKgjBpYFwMCruW86sOjv2lTpmzZF3mD3L/QIkX0Ode3Xc9MaMXFQnjiAK80xxkQ9rjTyjGrBWvbLwcxug4r9Akhmcxs/plCdYM=',
        '__VIEWSTATEGENERATOR':'B6E7D48B',
        '__EVENTVALIDATION':'2X6ageL+LlYeiTSgyQPd/FPAhPtg350MNiiKvURoS4xMsysX+0HyGjGN93yx7K27/NubbDHt2oTpWbFCv1dampTLrZkWvsf32lHlqJUliAsudoGhZ3kwM/XCxXSlvhNr',
        'Button1':'Button'
}

data = bytes(urllib.parse.urlencode(str),encoding='utf-8')
url = "http://192.168.21.195:8044/WebForm1"

#第二种写法
response = urllib.request.urlopen(url,data=data)
print(response.read().decode('utf-8'))

 

运行效果

 

再优化下正则去除多余html标签

import re

resstr=response.read().decode('utf-8')
new_st = re.sub(r'<[^>]*>','',resstr)
print(new_st.split())

　　

 

总结：1，测试后不需要添加头文件。

            2，抓数据方面,python语法比C#语法简单.

            3，python语法注意事项：data里面是键值队形式和json一致