.net爬虫了解一下

using System;
//添加selenium的引用
using OpenQA.Selenium.PhantomJS;
using OpenQA.Selenium.Chrome;

using OpenQA.Selenium.Support.UI;
using OpenQA.Selenium;
using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Net;
using HtmlAgilityPack;
namespace ConsoleApplication1
{
    class requests
    {

        public static string UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36";

        public static string AcceptLang = "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3";


        public static bool AddProxy = false;

        public static bool ProxyPort = false;
        public static string ProxyHost = null;
        private static string Status;
        private static string ResponseUri;
        private static HtmlDocument Document=new HtmlDocument();

        static PhantomJSDriver driver { get; set; }

        static void Main(string[] args)
        {
            //PhantomJSDriverService driverService = PhantomJSDriverService.CreateDefaultService();
            ////driverService.IgnoreSslErrors = true; //"any" also works
            //driverService.HideCommandPromptWindow = true;
            //driverService.ProxyType = "none";
            //driverService.SslProtocol = "any";
            //driverService.MaxDiskCacheSize = 1000;
            //driverService.DiskCache = true;
            //driverService.WebSecurity = false;
            //PhantomJSOptions phs = new PhantomJSOptions();
            //phs.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36");
            var req = get(new Uri("http://www.baidu.com"));
            var resp = response(req);
            var root = GetResult(resp);
            var nodelist=parse(root,"//div");
            Console.WriteLine(root.DocumentNode.InnerHtml);
            ChromeOptions options = new ChromeOptions();
            options.AddArgument("--headless");
            options.AddArgument("--nogpu");
            List<String> tagNmaeList = new List<string>(); ;
            //using (driver = new PhantomJSDriver(driverService, phs))
            //{
            using (var driver = new ChromeDriver(options))
            {
                driver.Manage().Window.Maximize();
                driver.Navigate().GoToUrl(@"https://qiita.com/ttake/items/2e470462f8d90e76e996");
                Thread.Sleep(5000);
                string source = driver.PageSource;
                Console.WriteLine(driver.Url);
                var html = driver.FindElement(By.TagName("html"));
                try
                {
                    var div = driver.FindElement(By.Id("global-nav"));

                }
                catch (NoSuchElementException)
                {
                    Console.WriteLine("找不到该元素"); ;
                }
                IJavaScriptExecutor js = (IJavaScriptExecutor)driver;


                var divnode = driver.FindElement(By.ClassName("highlight"));

                var bkgimg = js.ExecuteScript("arguments[0].style.getPropertyValue('background-image')", divnode);
                if (bkgimg == null)
                {
                    var outerhtml = divnode.GetAttribute("outerHTML");  //outerhtml
                    var outerText = divnode.GetAttribute("outerText");
                    var innerText = divnode.GetAttribute("innerText");
                    var children = divnode.GetAttribute("children");
                    var textContent = divnode.GetAttribute("textContent");
                    var childElementCount = divnode.GetAttribute("childElementCount");
                }

                var image = driver.FindElement(By.ClassName("it-Header_authorImage"));

                var width = image.Size.Width;
                var height = image.Size.Height;
                var elem = js.ExecuteScript("return document.defaultView.getComputedStyle(arguments[0])", image);

                Screenshot screenshot = ((ITakesScreenshot)driver).GetScreenshot();
                var dirpath = @"D:\temp";
                if (!Directory.Exists(dirpath))
                {

                    Directory.CreateDirectory(dirpath);

                }
                screenshot.SaveAsFile(@"D:\temp\seleniumScreenshot.png", ScreenshotImageFormat.Png);
            }
        }
        private static HttpWebRequest get(Uri uri)
        {
            HttpWebRequest httpWebRequest = WebRequest.Create(uri) as HttpWebRequest;

            if (httpWebRequest != null)
            {
                httpWebRequest.AllowAutoRedirect = true;
                httpWebRequest.AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate);
                httpWebRequest.UserAgent = UserAgent;
                if (!string.IsNullOrEmpty(AcceptLang))
                {
                    httpWebRequest.Headers.Add("Accept-Language", AcceptLang);
                }
                if (AddProxy)
                {
                    WebProxy proxy = new WebProxy(ProxyHost, ProxyPort);
                    httpWebRequest.Proxy = proxy;
                }
            }
            return httpWebRequest;
        }
        private static HttpWebResponse response(WebRequest request)
        {
            return (HttpWebResponse)request.GetResponse();
        }
        private static HtmlDocument GetResult(HttpWebResponse response)
        {
            if (response != null)
            {
                Status = response.StatusCode.ToString().ToUpper();
                ResponseUri = response.ResponseUri.AbsoluteUri;
                using (Stream responseStream = response.GetResponseStream())
                {
                    byte[] array = new byte[1024];
                    using (MemoryStream memoryStream = new MemoryStream())
                    {
                        int num;
                        do
                        {
                            num = responseStream.Read(array, 0, array.Length);
                            if (num > 0)
                            {
                                memoryStream.Write(array, 0, num);
                            }
                        }
                        while (num > 0);
                        memoryStream.Seek(0L, SeekOrigin.Begin);
                        Document.Load(memoryStream,Encoding.UTF8);
                    }
                }
            }
            return Document;
        }
        private static HtmlNodeCollection parse(HtmlDocument doc,string xpath)
        {

               var nodelist=doc.DocumentNode.SelectNodes(xpath);  
              return nodelist;
        
        }

    }
    
}