C#小爬虫，通过URL进行模拟发送接收数据
1 public async Task<string> SendDataAsync(HttpMethod httpMethod, string requestUrl, HttpContent postContent = null, string cookies = "") 2 { 3 HttpClientHandler httpHandler = new HttpClientHandler() 4 { 5 AllowAutoRedirect = true, 6 //CookieContainer = new CookieContainer(), 7 UseCookies = false, 8 AutomaticDecompression = DecompressionMethods.GZip 9 }; 10 11 using (HttpClient httpClient = new HttpClient(httpHandler)) 12 { 13 httpClient.DefaultRequestHeaders.ExpectContinue = false; 14 httpClient.DefaultRequestHeaders.Add("Accept", "*/*"); 15 httpClient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); 16 httpClient.DefaultRequestHeaders.Add("Accept-Language", " zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3"); 17 httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0"); 18 //var response = await httpClient.GetAsync(sUrl); 19 //return (await response.Content.ReadAsAsync<AvResult>()); 20 string sResult; 21 try 22 { 23 HttpRequestMessage request = new HttpRequestMessage(httpMethod, requestUrl); 24 bool isNullCookie = string.IsNullOrEmpty(cookies); 25 if (!isNullCookie) 26 { 27 //httpHandler.CookieContainer.SetCookies(new Uri(requestUrl), cookies); 28 request.Headers.Add("Cookie", cookies); 29 } 30 if (httpMethod.Method == "POST") 31 request.Content = postContent; 32 var response = await httpClient.SendAsync(request); 33 sResult = await response.Content.ReadAsStringAsync(); 34 } 35 catch (Exception ex) 36 { 37 sResult = ex.Message; 38 } 39 return sResult; 40 } 41 }
次上面的代码只是爬虫的模拟数据发送的部分代码，即自定义模拟浏览器客户端的，设置请求头，没有用Cookies进行登陆保存设置
