中信证券抓取（页码范围）

创建时间：2024年8月5日

一、完整代码

import re
import requests
from lxml import etree

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
}
url1 = 'http://www.cs.ecitic.com/newsite/cpzx/jrcpxxgs/zgcp/index.html'
res = requests.get(url=url1, headers=headers, verify=False)
res.encoding = res.apparent_encoding
total_num = int(re.search('var countPage = (?P<page>\d+)//共多少页', res.text).group('page'))


def get_data(url):
    url = url.replace('\n', '')
    print(url)
    response = requests.get(url=url,
                            headers=headers, verify=False)
    response.encoding = response.apparent_encoding
    data = response.text
    tree = etree.HTML(data)
    rows = tree.xpath('/html/body/div[4]/div/ul/li')

    for row in rows:
        row_mc = row.xpath('./span[@class="th1"]/text()')
        row_glr = row.xpath('./span[@class="th2"]/text()')
        row_fxpj = row.xpath('./span[@class="th3"]/text()')
        row_je = row.xpath('./span[@class="th4"]/text()')
        row_gs = row.xpath('./span[@class="th5"]/text()')
        sting = f"{row_mc}, {row_glr}, {row_fxpj}, {row_je}, {row_gs},'\n'"

        sting = sting.replace("['", "").replace("']", "").replace("'", "")
        with open('./zxzq/中信证券.txt', 'a+', encoding='utf-8') as f:
            f.write(sting)


start_page = int(input('请输入需要抓取的页码开始：（1开始）'))

end_page = int(input(f'请输入需要抓取的页码结束：（{total_num}结束）'))

for i in range(start_page - 1, end_page):
    if end_page + 1 > total_num:
        print('页码超过数据限制！！')
        exit(-1)
    if i == 0:
        url = 'http://www.cs.ecitic.com/newsite/cpzx/jrcpxxgs/zgcp/index.html'
        get_data(url)
    else:
        url = f'http://www.cs.ecitic.com/newsite/cpzx/jrcpxxgs/zgcp/index_{i}.html'
        get_data(url)

1.1 效果

二、学习点

2.1 verify 参数

verify关键字参数，在请求的时候不验证网站的ca证书

2.2 设置编码

res.encoding = res.apparent_encoding
# 大部分情况可以自动解码，实在不行可手动设置编码
res.encoding = 'utf-8'

2.3 运用正则表达式获取页码

total_num = int(re.search('var countPage = (?P<page>\d+)//共多少页', res.text).group('page'))
# search(pattern, string, flags=0)  扫描字符串寻找匹配的模式，返回一个match对象，如果没有找到匹配则返回None。

页面源代码对应的页码位置：

2.4 xpath路径

rows = tree.xpath('/html/body/div[4]/div/ul/li')

for row in rows:
    row_mc = row.xpath('./span[@class="th1"]/text()')
    row_glr = row.xpath('./span[@class="th2"]/text()')
    row_fxpj = row.xpath('./span[@class="th3"]/text()')
    row_je = row.xpath('./span[@class="th4"]/text()')
    row_gs = row.xpath('./span[@class="th5"]/text()')
    sting = f"{row_mc}, {row_glr}, {row_fxpj}, {row_je}, {row_gs},'\n'"

    sting = sting.replace("['", "").replace("']", "").replace("'", "")

先解析到所有的数据，然后在遍历匹配出来

2.5 分页逻辑

start_page = int(input('请输入需要抓取的页码开始：（1开始）'))

end_page = int(input(f'请输入需要抓取的页码结束：（{total_num}结束）'))

for i in range(start_page - 1, end_page):
    if end_page + 1 > total_num:
        print('页码超过数据限制！！')
        exit(-1)
    if i == 0:
        url = 'http://www.cs.ecitic.com/newsite/cpzx/jrcpxxgs/zgcp/index.html'
        get_data(url)
    else:
        url = f'http://www.cs.ecitic.com/newsite/cpzx/jrcpxxgs/zgcp/index_{i}.html'
        get_data(url)