利用Python爬取大学官网上边的精品文章（对初学者很友好）
一、需求分析
1.包含发布⽇期，作者，标题，阅读数以及正⽂。
2.可⾃动翻⻚。
3.范围：2020年内
二、实现代码
def import_mysql(data_list):
db = pymysql.connect(host='localhost', user='root', password='123456', port=3306, db='spiders', charset='utf8')
cursor = db.cursor()
# 如果数据表已经存在使用execute()方法删除表。
cursor.execute("DROP TABLE IF EXISTS novel")
sql = 'CREATE TABLE IF NOT EXISTS novel (' \
'date VARCHAR(255) NOT NULL, ' \
'title VARCHAR(255) NOT NULL, ' \
'author VARCHAR(255) NOT NULL, ' \
'read_count VARCHAR(255) NOT NULL,' \
'content VARCHAR(10000) NOT NULL)'
cursor.execute(sql)
table = 'novel'
for data in data_list:
keys = ', '.join(data.keys())
values = ', '.join(['%s'] * len(data))
sql = 'INSERT INTO {table}({keys}) VALUES ({values})'.format(table=table, keys=keys, values=values)
try:
if cursor.execute(sql, tuple(data.values())):
print('Successful import mysql')
db.commit()
except:
print('Failed import mysql')
db.rollback()
db.close()
def __init__(self):
self.__head = None
self.__node_be_inserted = None
self.__node_be_moved = None
#采用函数层次架构
#层次一：
def traverse_get_be_inserted_node(self): # find be inserted node ok
first_node = self.__head
bottom_child_node_list = []
def traverse_heap(current_node): # recursive function
if (current_node.get_child_node("left") == None) | (current_node.get_child_node("left") == None):
bottom_child_node_list.append(current_node)
if current_node.get_child_node("left") != None:
traverse_heap(current_node.get_child_node("left"))
if current_node.get_child_node("right") != None:
traverse_heap(current_node.get_child_node("right"))
traverse_heap(self.__head)
min_priority_number = 100000 # max priority
min_priority_node = None
for ele_instance in bottom_child_node_list:
if ele_instance.get_insert_priority() < min_priority_number:
min_priority_number = ele_instance.get_insert_priority()
min_priority_node = ele_instance
self.__node_be_inserted = min_priority_node
def value_float(self, newNode):# min value float ok
currentNode = newNode
currentNodeValue = currentNode.get_node_value()
currentParentNode = currentNode.get_parent_node()
if currentParentNode == None:
return "已经上浮到 root"
currentParentNodeValue = currentParentNode.get_node_value()
while currentNodeValue < currentParentNodeValue:
currentNode.set_node_value(currentParentNodeValue)
currentParentNode.set_node_value(currentNodeValue)
currentNode = currentParentNode
currentNodeValue = currentNode.get_node_value()
currentParentNode = currentNode.get_parent_node()
if currentParentNode == None:
#print("值已经上浮根节点值为{}".format(self.__head.get_node_value()))
return "已经上浮到root"
currentParentNodeValue = currentParentNode.get_node_value()
"""
doubleParentNode = parentNode.get_parent() # 2020.11.26 #交换 node_value 对象之间的链接关系不发生改变
def judge_left_or_right(parentNode, newNode):
if parentNode.get_child_node("left") == newNode:
return "left"
else:
return "right"
result_one = judge_left_or_right(parentNode, newNodeValue)
result_two = judge_left_or_right(doubleParentNode, parentNode)
"""
def traverse_get_be_moved_node(self): # ok
#改进方案：与traverse_get_be_inserted_node（）函数代码复用程度极高可用类继承减少复用
first_node = self.__head
bottom_child_node_list = []
def traverse_heap(current_node): # recursive function
if (current_node.get_child_node("left") == None) | (current_node.get_child_node("right") == None):## error ########
bottom_child_node_list.append(current_node)
if current_node.get_child_node("left") != None:
traverse_heap(current_node.get_child_node("left"))
if current_node.get_child_node("right") != None:
traverse_heap(current_node.get_child_node("right"))
traverse_heap(self.__head)
max_priority_number = -1 # max priority
max_priority_node = None
for ele_instance in bottom_child_node_list:
if ele_instance.get_insert_priority() > max_priority_number:
max_priority_number = ele_instance.get_insert_priority()
max_priority_node = ele_instance
self.__node_be_moved = max_priority_node
#print("被移动节点值为{}, priority值为{}".format(self.__node_be_moved.get_node_value(), self.__node_be_moved.get_insert_priority()))
def value_down(self):
current_node = self.__head
while current_node.get_child_node("left") != None and current_node.get_child_node("right") != None:
current_tuple = (current_node, current_node.get_node_value()) #current_node, current_node_value
current_left_child_node = current_node.get_child_node("left")
current_left_tuple = (current_left_child_node, current_left_child_node.get_node_value()) #current_left_child_node, current_left_child_node_value
current_right_child_node = current_node.get_child_node("right")
current_right_tuple = (current_right_child_node, current_right_child_node.get_node_value()) #current_right_child_node, current_right_child_node_value
current_tuple_list = [current_tuple, current_left_tuple, current_right_tuple]
sequence_current_tuple_list = []
for ele_tuple in current_tuple_list:
if len(sequence_current_tuple_list) == 0:
sequence_current_tuple_list.append(ele_tuple)
elif len(sequence_current_tuple_list) == 1:
if ele_tuple[1] < sequence_current_tuple_list[0][1]:
sequence_current_tuple_list.insert(0, ele_tuple)
else:
sequence_current_tuple_list.append(ele_tuple)
else: #len(sequence_current_tuple_list) == 2
if ele_tuple[1] < sequence_current_tuple_list[0][1]:
sequence_current_tuple_list.insert(0, ele_tuple)
elif ele_tuple[1] > sequence_current_tuple_list[1][1]:
sequence_current_tuple_list.append(ele_tuple)
else:
sequence_current_tuple_list.insert(1, ele_tuple)
current_node_index = sequence_current_tuple_list.index(current_tuple)
current_min_child_node = sequence_current_tuple_list[0][0]
if current_node_index >= 1:
current_min_child_node_value = current_min_child_node.get_node_value()
current_node = current_tuple[0]
current_node_value = current_tuple[1]
current_node.set_node_value(current_min_child_node_value)
current_min_child_node.set_node_value(current_node_value)
current_node = current_min_child_node
else:
return "value down successful"
if current_node.get_child_node("left") != None and current_node.get_child_node("right") == None:
current_node_value = current_node.get_node_value()
current_child_node = current_node.get_child_node("left")
current_child_node_value = current_child_node.get_node_value()
if current_node_value > current_child_node_value:
current_child_node.set_node_value(current_node_value)
current_node.set_node_value(current_child_node_value)
return "value down successful"
if current_node.get_child_node("right") != None and current_node.get_child_node("left") == None:
current_node_value = current_node.get_node_value()
current_child_node = current_node.get_child_node("right")
current_child_node_value = current_child_node.get_node_value()
if current_node_value > current_child_node_value:
current_child_node.set_node_value(current_node_value)
current_node.set_node_value(current_child_node_value)
return "value down successful"
return "value down successful"
#层次二
def insert(self, value):
if self.__head == None: # start first node
newNode = Node(value, None, 0)
self.__head = newNode
self.__node_be_inserted = newNode
else:
parent_node = self.__node_be_inserted
newNode = Node(value, parent_node, parent_node.get_insert_priority()+1)
if parent_node.get_child_node("left") == None:
parent_node.append_child_node("left", newNode)
elif parent_node.get_child_node("right") == None:
parent_node.append_child_node("right", newNode)
else:
print("得到的self.__node_be_inserted出现错误")
self.value_float(newNode)
self.traverse_get_be_inserted_node()
def deleMin(self):
if self.__head == None:
return None
if (self.__head.get_child_node("left") == None) and (self.__head.get_child_node("right") == None):
minValue = self.__head.get_node_value()
self.__head = None
return minValue
def judge_left_or_right(parentNode, childNode):
if parentNode.get_child_node("left") == childNode:
return "left"
else:
return "right"
min_node = self.__head
minValue = min_node.get_node_value()
self.traverse_get_be_moved_node()
be_moved_node = self.__node_be_moved
min_node.set_node_value(be_moved_node.get_node_value())
parentNode = be_moved_node.get_parent_node()
result = judge_left_or_right(parentNode, be_moved_node)
parentNode.delete_child_node(result)
self.value_down()
return minValue
#层次三
def buildHeap(self, _list):
_list.sort()
for i in _list:
self.insert(i)
#函数层次架构外
def findMin(self):
minValue = self.__head.get_node_value()
return minValue
@property
def isEmpty(self):
if self.__head == None:
return "True"
else:
return "False"
@property
def size(self):
first_node = self.__head
def traverse_heap(current_node): # recursive function
if current_node != None:
#print("执行一次，当前节点值为", current_node.get_node_value())
return 1 + traverse_heap(current_node. get_child_node("left")) + traverse_heap(current_node.get_child_node("right"))
else:
return 0
if first_node == None:
return 0
else:
return traverse_heap(first_node)
#递归方法递归堆中每一个节点求出堆中key的个数
三、效果展示
搞定，还望各位大佬不吝赐教~
