selenium爬去数据+存储
1 爬去数据代码
#coding=utf-8 from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC #加载TimeoutException模块,用于进行超时处理 from selenium.common.exceptions import TimeoutException #正则表达式 import re,sys from pyquery import PyQuery as pq from config import * #加载数据库操作模块 import mysqlOp driver=webdriver.Chrome() #使用phantomJs浏览器驱动 #driver=webdriver.PhantomJS() driver.get("https://www.taobao.com") driver.set_window_size(1400,900) wait=WebDriverWait(driver, 10) def search(): try: input=wait.until(EC.presence_of_element_located(By.CSS_SELECTOR,"#q")) submit=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#J_TSearchForm > div.search-button > button"))) input.clear() input.send_keys("美食") submit.click() #获取第一页的数据 get_goods() except TimeoutException : search() #获取总页码 def get_total(): #查找总页码 total=wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > div.total"))) return total.text #翻页 def next_page(page): try: input=wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.form > input"))) submit=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit"))) input.clear() input.send_keys(page) submit.click() wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > ul > li.item.active > span"),str(page))) #获取当前页的数据 count=get_goods() except TimeoutException: next_page(page) return count def get_goods(): wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,"#mainsrp-itemlist .items .item"))) #mainsrp-itemlist > div > div > div:nth-child(1) > div.item.J_MouserOnverReq.item-ad #mainsrp-itemlist > div > div > div:nth-child(1) html=driver.page_source doc=pq(html) items=doc("#mainsrp-itemlist .items .item").items() count=0 for item in items: goods={ 'image':item.find('.pic .img').attr('src'), 'price':item.find('.price').text(), 'deal' :item.find('.deal-cnt').text()[:-3], 'title':item.find('.title').text(), 'shop':item.find('.shop').text(), 'location':item.find('.location').text() } print(goods) #将数据插入数据库 mysqlOp.mysqlOp(goods) count+=1 return count def main(): search() total=get_total() #使用正则表达式提取页码 total=int(re.compile(r"(\d+)").search(total).group(1)) print(total) total_count=0 for i in range(2,total+1): count=next_page(i) total_count +=count print(total_count) if __name__=="__main__": main()
2 存入到mysql中
创建一个mysqlOp.py的文件
#coding=utf-8 from pymysql import * def mysqlOp(goods): conn=connect(host='127.0.0.1', port=3306, user='root', passwd='1qaz2wsx#EDC', db='taobao_meishi', charset='utf8') cursor=conn.cursor() cursor.execute("insert into goods(image,price,deal,title,shop,location) values(%s,%s,%s,%s,%s,%s)",(goods['image'],goods['price'],goods['deal'],goods['title'],goods['shop'],goods['location'])) conn.commit() cursor.close() conn.close()