DotnetSpider爬虫简单示例 net core
文章地址
https://blog.csdn.net/sD7O95O/article/details/78097556
安装爬虫框架 NUGET 安装DotnetSpider
创建HTTP协议数据包
var site = new Site { CycleRetryTimes = 1, SleepTime = 200, Headers = new Dictionary<string, string>() { {"Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" }, {"Accept-Encoding","gzip, deflate, br" }, {"Accept-Language","zh-CN,zh;q=0.9" }, {"X-Requested-With","XMLHttpRequest" }, { "Referer", "https://blog.csdn.net/sD7O95O/article/details/78096027"}, { "Connection","keep-alive" }, { "Content-Type","text/html; charset=UTF-8" }, { "Host","blog.csdn.net"}, { "User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"} } };
site.AddStartUrl(surl);
创建一个爬虫
Spider spider = Spider.Create(site, // new QueueDuplicateRemovedScheduler(), new CsdnArticleProcessor() //这个自定义的处理器 ).AddPipeline(new CsdnArticlePipeline()); //自定义管道
设置爬虫
spider.Downloader = new HttpClientDownloader(); spider.ThreadNum = 1; spider.EmptySleepTime = 3000;
运行
spider.Run();
CsdnArticleProcessor 处理器 xpath 分析就不贴了
public class CsdnArticleProcessor : BasePageProcessor { protected override void Handle(Page page) { //page.Selectable.SelectList(Selectors.XPath("//table[@id='ip_list']/tr[2]/td[2]/text()")).Nodes(); // 利用 Selectable 查询并构造自己想要的数据对象 //xpath 得到自己需要的数据 //... //page.AddResultItem("CountryResult", results);放入管道 } }
class CsdnArticlePipeline : BasePipeline { public override void Process(IEnumerable<ResultItems> resultItems, ISpider spider) { var conlist = resukt.GetResultItem("CountryResult");获取数据
//DB处理
} }