没来做翻页，只爬第一页
import logging
import requests
import bs4
import json
import psycopg2
from io import StringIO
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s- %(message)s')
blist = []
def main():
url = 'https://www.qidian.com/finish/'
html = request_douban(url)
soup = bs4.BeautifulSoup(html,'lxml')
save_to_txt(soup)
def request_douban(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/88.0.4324.146 Safari/537.36',
}
try:
response = requests.get(url,headers=headers)
if response.status_code == 200:
return response.text
except requests.RequestException:
return None
def save_to_txt(soup):
booklist = soup.find(class_="main-content-wrap fl").find_all("li")
logging.info(booklist)
for i in booklist:
logging.info(i.find('h4'))
try:
shuming = i.find('h4').string
zuozhe = i.find(class_="name").string
leixing = i.find(class_="go-sub-type").string
zhuangtai = i.find(class_="author").find('span').string
dizhi = 'https:' + i.find('h4').find( 'a', href=True)['href']
tap = (shuming, zuozhe, leixing, zhuangtai, dizhi)
blist.append('\t'.join('%s' % idd for idd in tap))
logging.info(list)
writedatabase(list)
except:
return None
def writedatabase(blist):
with open("config/kafka_and_postgres_config.json") as json_file:
config = json.load(json_file)
Conf = json.loads(json.dumps(config))
pghost = Conf['postgres_host']
pguser = Conf['postgres_user']
pgpassword = Conf['postgres_password']
pgdatabase = Conf['postgres_database']
pgtable = Conf['postgres_table']
s = ""
for action in blist:
s += action + '\n'
logging.info(s)
conn = psycopg2.connect(host=pghost, user=pguser, password=pgpassword, database=pgdatabase)
cur = conn.cursor()
cur.copy_from(StringIO(s), pgtable, columns=('shuming', 'zuozhe', 'leixing', 'zhuangtai', 'dizhi'))
conn.commit()
cur.close()
conn.close()
list = []
print('完成')
if __name__=="__main__":
main()
