Python爬虫，爬取实验楼全部课程
目的：
使用requests库以及xpath解析进行实验楼所有课程，存入MySQL数据 库中。
准备工作：
首先安装，requests库，lxml库，以及peewee库。在命令行模式，使用以下命令。
pip install requests pip install lxml pip install peewee
然后，就可以打开编辑器编写代码了。
代码:
1 # 引入所需要的库 2 import time 3 import requests 4 from peewee import * 5 from lxml import etree 6 # 这个程序要倒着看 7 8 # 这个是连接数据库的，名字和密码根据自己情况修改 9 db = MySQLDatabase('shiyanlou', user='root', passwd='xxxxxx') 10 11 12 class Course(Model): 13 title = CharField() 14 teacher = CharField() 15 teacher_courses = IntegerField() 16 tag = CharField() 17 study_num = IntegerField() 18 content = CharField() 19 20 class Meta: 21 database = db 22 23 24 Course.create_table() 25 26 27 def parse_content(url, title, tag, study_num): 28 print('课程地址：' + url) 29 res = requests.get(url) 30 xml = etree.HTML(res.text) 31 # 获取页面里的简介 32 try: 33 content = xml.xpath('//meta[@name="description"]/@content')[0] 34 except Exception as e: 35 content = '无' 36 # 获取老师名字 37 try: 38 teacher = xml.xpath( 39 '//div[@class="sidebox mooc-teacher"]//div[@class="mooc-info"]/div[@class="name"]/strong/text()')[0] 40 except Exception as e: 41 teacher = '匿名' 42 # 获取老师发表课程数目 43 try: 44 teacher_courses = xml.xpath( 45 '//div[@class="sidebox mooc-teacher"]//div[@class="mooc-info"]/div[@class="courses"]/strong/text()')[0] 46 except Exception as e: 47 teacher_courses = '未知' 48 # 存入数据库 49 try: 50 course = Course(title=title, teacher=teacher, 51 teacher_courses=int(teacher_courses), tag=tag, study_num=int(study_num), content=content) 52 course.save() 53 except Exception as e: 54 print('一条数据存取失败') 55 56 57 def get_course_link(url): 58 # 获取每一页的信息，传给下一个函数 59 response = requests.get(url) 60 xml = etree.HTML(response.text) 61 # contains()是包含的意思 62 courses = xml.xpath( 63 '//div[contains(@class, "col-md-3") and contains(@class, "col-sm-6") and contains(@class, "course")]') 64 for course in courses: 65 try: 66 url = 'https://www.shiyanlou.com' + course.xpath('.//a/@href')[0] 67 except Exception as e: 68 print('一个课程页面未获得') 69 continue 70 title = course.xpath('.//div[@class="course-name"]/text()')[0] 71 study_people = course.xpath( 72 './/span[@class="course-per-num pull-left"]/text()')[1].strip() 73 # study_people = int(study_people) 74 try: 75 tag = course.xpath( 76 './/span[@class="course-money pull-right"]/text()')[0] 77 except Exception as e: 78 tag = "普通" 79 parse_content(url=url, title=title, tag=tag, study_num=study_people) 80 # time.sleep(0.5) 81 82 83 def main(): 84 # 通过requests库的get获得目标地址的返回信息，类型为Response 85 response = requests.get('https://www.shiyanlou.com/courses/') 86 # 将返回信息的文本转化为xml树，可以通过xpath来进行查询 87 xml = etree.HTML(response.text) 88 # 由分析网页源代码可以总结，url分页模式，只有最后的数字不一样 89 course_link = 'https://www.shiyanlou.com/courses/?category=all&course_type=all&fee=all&tag=all&page={}' 90 # 这里获得最大页数就可以了,xpath()函数里的便是寻找路径了 91 # //会在全文来进行查找，//ul则是查找全文的ul标签，//ul[@class="pagination"]会仅查找有class属性， 92 # 且为"pagination"的标签，之后/li是查找当前的ul标签下的li标签（仅取一层),取查询到的列表倒数第二个标签 93 # 为li[last()-1],/a/text()查询a标签里的文本内容 94 page = xml.xpath('//ul[@class="pagination"]/li[last()-1]/a/text()') 95 if len(page) != 1: 96 print('爬取最大页数时发生错误！！') 97 return None 98 # page原是一个列表，这里取出它的元素，并转化为Int型 99 page = int(page[0]) 100 # 将每一页的url传给get_course_link函数进行处理 101 for i in range(1, page + 1): 102 # 填入course_link,获取完整url 103 url = course_link.format(i) 104 print('页面地址：' + url) 105 # 调用另一个函数 106 get_course_link(url) 107 108 109 if __name__ == '__main__': 110 # 调用main函数 111 main() 112 113 # [Finished in 218.5s]
