dogedoge浏览器爬取标题
# coding:utf-8 import hashlib import datetime import lxml import pymysql import requests from lxml import etree import sys reload(sys) sys.setdefaultencoding('utf-8') def search_data(kw, n): ll = [] res = requests.get('https://www.dogedoge.com/results?q={}'.format(kw)) if n > 1: res = requests.get('https://www.dogedoge.com/results?q={}&p={}'.format(kw, n)) con = etree.HTML(res.text) url = con.xpath('//div[@class="result results_links_deep highlight_d result--url-above-snippet"]') for u in url: title = '' for i in u.xpath('./div/h2/a//text()'): title += i url = '' for i in u.xpath('./div/div/div/a/span//text()'): url += i domain = '' if url.find('http') != -1: domain = url.split('/')[2] else: domain = url.split('/')[0] md5 = hashlib.md5(url).hexdigest() item = {} item['keywd'] = kw item['domain'] = domain item['title'] = title item['md5'] = md5 item['url'] = url item['searcher'] = 'dogedoge' ll.append(item) save(ll) try: next = con.xpath('//div[@id="rld-2"]') except: print '没有下一页了' return '' else: return next def main(kw): n = 1 while True: next_page = search_data(kw, n) if not next_page: break n += 1 def save(ll): db = pymysql.connect( host=MYSQL_HOST, db=MYSQL_DBNAME, user=MYSQL_USER, passwd=MYSQL_PASSWD, charset='utf8', use_unicode=True) cursor = db.cursor() for item in ll: # print type(item), item['searcher'] try: # 插入数据库 cursor.execute( "insert into weixintb(md5,keyword,title,url,`date`,`domain`, browser) value(%s, %s, %s, %s, %s, %s,%s)", (item['md5'], item['keywd'], item['title'], item['url'], datetime.datetime.now(), item['domain'], item['searcher'] )) # 提交sql语句 db.commit() except Exception as error: # 出现错误时打印错误日志 # print error # logger.error(error) db.rollback() cursor.close() db.close() main('爬取关键词')