爬虫数据存储csv

爬虫数据存储csv

一，csv文件的简单读写

import csv

# CSV也叫逗号分分隔，一般以逗号分隔，也可以使用空格或者tab分隔
csv_file = open("file/test.csv", "w+")
# 写入
try:
    csv_write = csv.writer(csv_file)
    csv_write.writerow(("col1", "col2", "col3"))
    for i in range(4):
        csv_write.writerow((i*1, i*10, i*100))
finally:
    csv_file.close()

# 读出，当然上面写入也可以使用with as语法
with open("file/test.csv", "r") as f:
    reader = csv.reader(f)
    for i in reader:
        print(i)

二， 爬虫实战，爬取世界大学排名存入csv中

import requests
from bs4 import BeautifulSoup
import csv
"""
https://www.dxsbb.com/news/16131.html
获取本网站上的世界大学1000名。因为时表格，存入csv文件
"""


def wirte_csv(data_list):
    # 之前为gb2312编码，csv写入部分字符无法表示，所以转换为utf-8编码
    csv_file = open("file/university.csv", "w+", encoding="utf-8")
    try:
        csv_writer = csv.writer(csv_file)
        for i in data_list:
            # 因为之前使用了\t来拼接，所以这次使用'\t'来就行分割成为列表
            csv_writer.writerow(i.split('\t')[:-1])
    finally:
        csv_file.close()


def get_soup(url, encode):
    # 获取soup对象
    header = {
        'User-Agent': 'python-requests/2.22.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'
    }
    resp = requests.get(url=url, headers=header)
    resp.encoding=encode
    return BeautifulSoup(resp.text, "lxml")


def get_university(soup):
    # 获取数据
    un_list = soup.find("table").find_all("tr")
    td_list = []
    for un_row in un_list:
        td_row=""
        for un_td in un_row.find_all("td"):
            td_row = td_row+un_td.text+"\t"
        td_list.append(td_row)
    return td_list

if __name__ == '__main__':
    url = r"https://www.dxsbb.com/news/16131.html"
    # 获取bs对象，并且此网站使用gb2312编码
    soup = get_soup(url,"gb2312")
    list = get_university(soup)
    wirte_csv(list)

结果:生成文件university.csv，打开即可看到将爬取的数据。