.Net Core爬虫爬取妹子网图片

现在网上大把的Python的爬虫教程，很少看见有用C#写的，正好新出的.Net Core可以很方便的部署到Linux上，就用妹子图做示范写个小爬虫

在C#下有个很方便的类库 HtmlAgilityPack 可以用来分析网页

我们先新建一个.Net Core控制台项目MzituCrawler，在Nuget管理器添加HtmlAgilityPack的引用 Install-Package HtmlAgilityPack -Version 1.9.2 

我们打开妹子图的网页，点击下面的翻页按钮，发现每页的地址有个固定的格式 https://www.mzitu.com/page/页码/ 

我们先获取总共多少页

var baseUrl = $"https://www.mzitu.com";
HtmlWeb web = new HtmlWeb();
var indexDoc = web.Load(baseUrl);
var pageNode = indexDoc.DocumentNode.SelectNodes("/html/body/div[@class='main']/div[@class='main-content']/div[@class='postlist']/nav/div/a").Last(a => a.GetAttributeValue("class", string.Empty) == "page-numbers");
var pageCount = int.Parse(pageNode.InnerText);

查看网页上链接的元素可以看到每个链接对应的xpath地址为 //*[@id='pins']/li/a 

我们用HtmlAgilityPack获取第每一页的内容

for (int pageIndex = 1; pageIndex <= pageCount; pageIndex++)
{
    var url = new Uri(new Uri(baseUrl), $"/page/{pageIndex}/").ToString();
    var doc = web.Load(url);
    var nodes = doc.DocumentNode.SelectNodes("//*[@id='pins']/li/a");
    if (nodes.Count > 0x0)
    {
        foreach (var node in nodes)
        {
            var title = node.SelectSingleNode("img").GetAttributeValue("alt", string.Empty);
            var href = node.GetAttributeValue("href", string.Empty);
            href = new Uri(new Uri(baseUrl), href).ToString();
            DownloadImages(downloadFolder: Path.Combine(baseFolder, title), url: href);
        }
    }
    else
    {
        return;
    }
}

其中方法 DownloadImages 是下载对应链接里面图片的方法

private static void DownloadImages(string downloadFolder, string url)
{
    if (!Directory.Exists(downloadFolder))
    {
        Directory.CreateDirectory(downloadFolder);
    }
    HtmlWeb web = new HtmlWeb();
    var indexDoc = web.Load(url);
    var pageNode = indexDoc.DocumentNode.SelectNodes("/html/body/div[@class='main']/div[@class='content']/div[@class='pagenavi']/a").Reverse().Skip(1).First();
    var pageCount = pageNode == null ? 1 : int.Parse(pageNode.InnerText);
    for (int pageIndex = 1; pageIndex <= pageCount; pageIndex++)
    {
        var doc = web.Load($"{url}/{pageIndex}");
        var imageNode = doc.DocumentNode.SelectSingleNode("/html/body/div[2]/div[1]/div[3]/p/a/img");
        if (imageNode != null)
        {
            var imageUrl = imageNode.GetAttributeValue("src", string.Empty);
            imageUrl = new Uri(new Uri(url), imageUrl).ToString();
            if (historyUrl.Contains(imageUrl))
            {
                continue;
            }
            using (var client = new HttpClient())
            {
                client.DefaultRequestHeaders.Host = "i.meizitu.net";
                client.DefaultRequestHeaders.Pragma.ParseAdd("no-cache");
                client.DefaultRequestHeaders.AcceptEncoding.ParseAdd("gzip, deflate");
                client.DefaultRequestHeaders.AcceptLanguage.ParseAdd("zh-CN,zh;q=0.8,en;q=0.6");
                client.DefaultRequestHeaders.CacheControl = new System.Net.Http.Headers.CacheControlHeaderValue { NoCache = true };
                client.DefaultRequestHeaders.Connection.ParseAdd("keep-alive");
                client.DefaultRequestHeaders.Referrer = new Uri(url);
                client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
                client.DefaultRequestHeaders.Accept.ParseAdd("image/webp,image/apng,image/*,*/*;q=0.8");
                var buffer = client.GetByteArrayAsync(imageUrl).Result;
                var fileName = new Uri(imageUrl).Segments.Last();
                File.WriteAllBytes(Path.Combine(downloadFolder, fileName), buffer);
            }
        }
    }
}