网站添加robots.txt和sitemap.xml

robots.txt在线生成：

robots文件生成 在线网站robots.txt文件生成器

robots.txt:

# robots.txt
User-agent: *
Disallow: 
Disallow: /cgi-bin/
Sitemap: http://blog.xutongbao.top/sitemap.xml

使用node生成sitemap.xml文件：

const { SitemapStream, streamToPromise } = require('sitemap')
const { Readable } = require('stream')

//遍历文件夹获取link列表
const getLinks = async () => {
  return await new Promise((resolve, rejects) => {
    fs.readdir(outputDir, function (err, files) {
      if (err) {
        console.warn(err)
      } else {
        let links = files.map(item => {
          
          return { url: `http://blog.xutongbao.top/blog/src/md/${item.replace('md', 'html')}`, changefreq: 'daily', priority: 0.3 }
        })
        resolve(links)
      }
    })
  })
}

//根据link列表生成sitemap.xml文件
const getSitemap = async (req, res) => {
  const links = await getLinks()
  //const links = [{ url: '/page-1/', changefreq: 'daily', priority: 0.3 }]

  // Create a stream to write to
  const stream = new SitemapStream({ hostname: 'http://blog.xutongbao.top' })

  // Return a promise that resolves with your XML string
  let mySitemap = '1'
  await streamToPromise(Readable.from(links).pipe(stream)).then((data) => {
    mySitemap = data.toString()
  })
  const sitemapFilePath = './csdn/sitemap.xml'
  fs.writeFile(sitemapFilePath, mySitemap, { encoding: 'utf8' }, (err) => {})

  res.send({
    state: 1,
    data: {
      mySitemap,
      links
    },
    message: 'sitemap'
  })
}

sitemap.xml:

<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">
  <url>
    <loc>http://blog.xutongbao.top/blog/src/md/2017-08-24_17_39_00.html</loc>
    <changefreq>daily</changefreq>
    <priority>0.3</priority>
  </url>
  <url>
    <loc>http://blog.xutongbao.top/blog/src/md/2017-08-24_20_01_00.html</loc>
    <changefreq>daily</changefreq>
    <priority>0.3</priority>
  </url>
</urlset>