批量下载小说网站文章

 

 

using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;

namespace DownloadNovel
{
    class Program
    {
        static void Main(string[] args)
        {
            //小说网站的网址
            string webSiteUrl = "http://www.biqugew.com";
            //指定小说目录的网址
            string NovelUrl = "http://www.biqugew.com/book/10/";
            
           

            DownNovel(webSiteUrl, NovelUrl);
        }
        private static void DownNovel(string webSiteUrl, string NovelUrl)
        {
            string[] split = { "<br>", "\r\n" };
            //指定小说的目录的 Xpath
            string TableXpath = "/body[1]/div[1]/div[5]/div[1]/dl[1]/dd";
            //获取小说标题的 XPath
            string TitleXpath = "/html[1]/body[1]/div[1]/div[3]/div[1]/div[2]/h1[1]";
            //获取指定小说的内容的 Xpath
            string ContentsXpath = "/html/body/div/div[3]/div/div[3]";
            WebClient client = new WebClient { Encoding = Encoding.GetEncoding("GB2312") };
            HtmlNodeCollection nodes = null;
            {
                HtmlDocument doc = new HtmlDocument();
                //获取目录页
                doc.LoadHtml(client.DownloadString(NovelUrl));
                nodes = doc.DocumentNode.SelectNodes(TableXpath);
            }
            //解析目录页
            foreach (HtmlNode node in nodes)
            {
                HtmlDocument doc = new HtmlDocument();
                //获取小说单章的网站
                string url = webSiteUrl + node.SelectSingleNode("a").Attributes["href"].Value;
                //获取小说单章整个网页
                doc.LoadHtml(client.DownloadString(url));
                //获取本章小说的标题
                string title = doc.DocumentNode.SelectSingleNode(TitleXpath).InnerHtml;
                //获取小说文本内容 doc.DocumentNode.SelectSingleNode("/html/body/div/div[3]/div/div[3]").OuterHtml
                string str = doc.DocumentNode.SelectSingleNode(ContentsXpath).InnerHtml.Replace("&nbsp;", "");
                //过滤文本中的特殊字符和字符串
                string aticale = "";
                foreach (var txt in str.Split(split, StringSplitOptions.RemoveEmptyEntries))
                {
                    if (!txt.Contains("<a"))
                        aticale += txt;
                }
                Console.WriteLine(title);
                WriteLog(title + Environment.NewLine + aticale);
            }
            
        }

        static void WriteLog(string msg)
        {
            string path = Environment.CurrentDirectory + "/novel/";
            if (!Directory.Exists(path)) Directory.CreateDirectory(path);
            string fileName = DateTime.Now.ToString("yyyy-MM-dd");
            string filepath = path + fileName + ".txt";
            Stream fileStream = File.Open(filepath, FileMode.Append, FileAccess.Write, FileShare.Write);
            StreamWriter writeAdapter = new StreamWriter(fileStream, Encoding.Default);
            writeAdapter.WriteLine(msg);
            writeAdapter.WriteLine();
            writeAdapter.Close();
        }

    }
}