使用Phantomjs和ChromeDriver添加Cookies的方法
一、查看代码 :
namespace ToutiaoSpider { class Program { static void Main(string[] args) { var db = Db.GetDataHelper(); db.CreateOrUpdateTable(typeof(Models.ArticleMonitorModule)); while (true) { var src = LoadSource(); try { //var item = GetItem(src); //if (item != null) //{ // try // { // db.Insert(item.ToArray()); // Console.WriteLine("Insert a row in :" + DateTime.UtcNow.AddHours(8).ToString("yyyy-MM-dd HH:mm:ss")); // } // catch (Exception ex) // { Console.WriteLine("Insert to database error:" + ex.Message); } //} if (!Directory.Exists("C:\\TempFiles\\")) Directory.CreateDirectory("C:\\TempFiles\\"); File.WriteAllText("C:\\TempFiles\\" + DateTime.Now.ToString("yyyy_MM_dd_HH_mm_ss") + ".html", src); } catch (Exception ex) { Console.WriteLine("Get item error:" + ex.Message); } Thread.Sleep(5 * 60 * 1000); } } static List<Models.ArticleMonitorModule> GetItem(string src) { List<Models.ArticleMonitorModule> lst = new List<ArticleMonitorModule>(); HtmlAgilityPack.HtmlDocument docs = new HtmlAgilityPack.HtmlDocument(); docs.LoadHtml(src); var jsonText = docs.DocumentNode.InnerText; var json = (JObject)JsonConvert.DeserializeObject(jsonText); if (json["message"].Value<string>() == "success") { var token = json.GetValue("data"); if (token == null) return lst; foreach (var data in token) { var title = data["title"]?.Value<string>(); //if (title != "中国与巴拿马建立外交关系") continue; var go_detail_count = data["go_detail_count"]?.Value<int>(); var url = data["display_url"]?.Value<string>(); var itemId = data["item_id"]?.Value<string>(); var comments_count = data["comments_count"]?.Value<int>(); var site = "toutiao.com"; ArticleMonitorModule item = new ArticleMonitorModule() { __id = Guid.NewGuid(), article_id = itemId, site = site, url = url, comment_count = comments_count, visit_count = go_detail_count, fetch_time = DateTime.UtcNow.AddHours(8), like_count = null }; lst.Add(item); } } return lst; } static IWebDriver CreateDriver() { var service = PhantomJSDriverService.CreateDefaultService(); var driver = new OpenQA.Selenium.PhantomJS.PhantomJSDriver(service, new PhantomJSOptions(), TimeSpan.FromSeconds(120)); //var service = ChromeDriverService.CreateDefaultService(); //var driver = new ChromeDriver(service, new ChromeOptions(), TimeSpan.FromSeconds(120)); //var service = OpenQA.Selenium.Firefox.FirefoxDriverService.CreateDefaultService(); //var driver = new OpenQA.Selenium.Firefox.FirefoxDriver(service, new OpenQA.Selenium.Firefox.FirefoxOptions(), TimeSpan.FromSeconds(120)); return driver; } static string LoadSource() { var driver = CreateDriver(); var nav = driver.Navigate(); var cookies = driver.Manage().Cookies; //nav.GoToUrl("http://www.toutiao.com/c/user/favourite/?page_type=2&user_id=61045799395&max_behot_time=0&count=20&as=A15569B3CF98ED7&cp=593F781EDDB7FE1&max_repin_time=0"); int retryCount = 6; while (true) { try { //nav.GoToUrl("http://www.toutiao.com/c/user/favourite/?page_type=2&user_id=61045799395&max_behot_time=0&count=20&as=A14529438F1A7A4&cp=593F3A47DAD44E1&max_repin_time=0"); var url = "http://is.snssdk.com/2/article/information/v21/?version_code=6.1.6&app_name=news_article&vid=C5585644-2731-495E-8CF2-B42BBA4D7780&device_id=35980279488&channel=App%20Store&resolution=1125*2001&aid=13&ab_version=120431,134942,136400,126064,122834,130106,126068,128826,134127,136030,137117,136268,137571,126070,136111,116022,135623,125502,137069,125174,135489,133019,137083,126059,137452,135631,136930,122948,137474,137431,31210,133013,135290,131207,114338,133770&ab_feature=z1&openudid=f870822c71509e95ee8f58db8b1d70ce9cb14713&live_sdk_version=1.6.5&idfv=C5585644-2731-495E-8CF2-B42BBA4D7780&ac=WIFI&os_version=10.3.2&ssmix=a&device_platform=iphone&iid=11267657395&ab_client=a1,f2,f7,e1&device_type=iPhone%206S%20Plus&idfa=B1742B5B-DF14-44EF-A325-362873389ABA&aggr_type=1&article_page=0&device_id=35980279488&from_category=news_entertainment&group_id=6411002681368035586"; nav.GoToUrl(url); var uri = new Uri(url); cookies.AddCookie(new Cookie("csrftoken", "b9e36219cad78dfe6a1c687d6b368b52", uri.DnsSafeHost, "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("csrftoken", "b9e36219cad78dfe6a1c687d6b368b52", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("CNZZDATA1259612802", "2103889297-1495413998-null%7C1495413998", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("UM_distinctid", "15c2dec3e02f09-0fc740fef2ffb7-572f7b6e-1fa400-15c2dec3e03cf0", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("tt_webid", "6431015020234769922", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("uuid", "\"w:a5e3254676244e0ab15fc4291e372d14\"", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("_ga", "GA1.2.1639521857.1495419078", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("_gid", "GA1.2.396214455.1495419159", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("sso_login_status", "1", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("login_flag", "f8947cb01c5a760d0cbc4925e601ca60", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("sessionid", "5c865f7a96598cff3b3d580fcd3dfd27", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("sid_tt", "5c865f7a96598cff3b3d580fcd3dfd27", "toutiao.com", "/", DateTime.Now.AddYears(1))); //cookies.AddCookie(new Cookie("sid_guard", "\"5c865f7a96598cff3b3d580fcd3dfd27|1495419148|2591999|Wed\054 21-Jun-2017 02:12:27 GMT\"", "toutiao.com", "/", DateTime.Now.AddYears(1))); nav.Refresh(); Console.WriteLine("OK!" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); break; } catch (Exception ex) { nav.Refresh();//说明,对于PhantomJS的话,虽然前面AddCookies时总是抛出异常,但是实际Cookies是添加成功的。所以需要在此Refresh。 Console.WriteLine("retry " + retryCount + ":" + ex.Message); Thread.Sleep(1000); if (retryCount-- <= 0) break; } } var pageSource = driver.PageSource; //var st = driver.TakeScreenshot(); //st.SaveAsFile("d:\\" + DateTime.Now.ToString("yyyyMMddHHmmssfff") + ".jpg", ScreenshotImageFormat.Jpeg); driver.Close(); driver.Dispose(); return pageSource; } }
二、代码解说:
在Main方法中调用了LoadSource方法,LoadSource方法中第1步先调用CreateDriver方法创建一个Driver,例子中创建了一个Phantomjs的Driver,然后通过driver.Navigate()取得Navigate对像,然后调用 GotoURL转到一个URL,之后才能给driver.Manager().Cookies中添加Cookie,添加完Cookies之后,调用nav的Refresh方法,即可重新发起携带Cookies的请求。不过这里有个小问题,就是添加Cookie这一步,当使用Chrome时,在GoToURL之后,即可直接成功添加Cookie,但是在Phantomjs中总是AddCookie时抛出异常。这时候查看官方文档,官方文档说,虽然这里抛出异常,但是Cookies依然是成功添加的。所以我们在Catch到异常之后,调用了一下Refresh去刷新。这时使用Fiddler检测,果然携带了Cookie,以上代码写的非常乱,因为只是想简单快速测试Demo,所以代码非常乱。
三、如图:
宋兴柱:转载内容,请标明出处,谢谢!源文来自 宝贝云知识分享:https://www.dearcloud.cn