爬下淘宝
1：准备
安装chrome 浏览器；
下载chromedriver， 使用vim 替换chromedriver中的$cdc_asdjflasutopfhvcZLmcfl_，字符串长度不变，vim 使用/cdc_模式
安装python3 , pip3， 使用pip安装selenium
2：代码
# -*- coding: utf-8 -*- from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver import ActionChains from selenium.webdriver.common.keys import Keys from pyquery import PyQuery as pq from time import sleep import es #定义一个taobao类 class taobao_infos: #对象初始化 def __init__(self): url = 'https://login.taobao.com/member/login.jhtml' self.url = url chrome_options = webdriver.ChromeOptions() # 设置浏览器初始 位置x,y & 宽高x,y chrome_options.add_argument(f'--window-position={217},{172}') chrome_options.add_argument(f'--window-size={1200},{1000}') # 关闭自动测试状态显示 // 会导致浏览器报：请停用开发者模式 # window.navigator.webdriver还是返回True,当返回undefined时应该才可行。 # 修改chromedriver ，替换$cdc_asdjflasutopfhvcZLmcfl_，字符串长度不变 chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"]) # 关闭开发者模式 chrome_options.add_experimental_option('useAutomationExtension', False) # 禁止图片加载 chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2}) # 设置中文 chrome_options.add_argument('lang=zh_CN.UTF-8') # 更换头部 chrome_options.add_argument( 'user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"') # 隐藏浏览器 # chrome_options.add_argument('--headless') # 部署项目在linux时，其驱动会要求这个参数 chrome_options.add_argument('--no-sandbox') self.browser = webdriver.Chrome(executable_path=chromedriver_path, options=chrome_options) # 设置执行js代码转换模式 self.browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", { "source": """ Object.defineProperty(navigator, 'webdriver', { get: () => undefined }) """ }) #超时时长为10s self.wait = WebDriverWait(self.browser, 10) self.es = es.es_storage(['http://127.0.0.1:9200']) #延时操作,并可选择是否弹出窗口提示 def sleep_and_alert(self,sec,message,is_alert): for second in range(sec): if(is_alert): alert = "alert(\"" + message + ":" + str(sec - second) + "秒\")" self.browser.execute_script(alert) al = self.browser.switch_to.alert sleep(1) al.accept() else: sleep(1) #登录淘宝 def login(self): # 打开网页 self.browser.get(self.url) # 等待页面加载完 self.browser.implicitly_wait(30) # 输入密码并点击登入 self.wait.until(EC.presence_of_element_located((By.ID, 'fm-login-id'))).send_keys(username) self.wait.until(EC.presence_of_element_located((By.ID, 'fm-login-password'))).send_keys(password) self.browser.find_element_by_class_name('fm-btn').click() self.verify() sleep(3) # 直到获取到淘宝会员昵称才能确定是登录成功 taobao_name = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.site-nav-bd > ul.site-nav-bd-l > li#J_SiteNavLogin > div.site-nav-menu-hd > div.site-nav-user > a.site-nav-login-info-nick '))) # 输出淘宝昵称 print(taobao_name.text) def verify(self): try: slider = self.wait.until(EC.presence_of_element_located((By.ID, 'nc_1_n1z'))) ActionChains(self.browser).click_and_hold(on_element=slider).perform() ActionChains(self.browser).move_by_offset(xoffset=280, yoffset=0).perform() ActionChains(self.browser).release().perform() sleep(1) self.browser.find_element_by_class_name('fm-btn').click() except Exception as e: print("点击滑块验证失败 %s" % e) else: print("尝试输入手机验证码") sleep(30) # 获取天猫商品总共的页数 def search_toal_page(self): good_total = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div#mainsrp-pager > div.m-page.g-clearfix > div.wraper > div.inner.clearfix > div.total'))) page_total = good_total.text.replace("共","").replace("页，","").replace("，","") return page_total # 翻页操作 def next_page(self, page_number): # 等待该页面input输入框加载完毕 input = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div#mainsrp-pager > div.m-page.g-clearfix > div.wraper > div.inner.clearfix > div.form > input.input.J_Input'))) # 等待该页面的确定按钮加载完毕 submit = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div#mainsrp-pager > div.m-page.g-clearfix > div.wraper > div.inner.clearfix > div.form > span.btn.J_Submit'))) # 清除里面的数字 input.clear() # 重新输入数字 input.send_keys(page_number) # 强制延迟1秒，防止被识别成机器人 sleep(1) # 点击确定按钮 submit.click() # 模拟向下滑动浏览 def swipe_down(self,second): for i in range(int(second/0.1)): js = "var q=document.documentElement.scrollTop=" + str(300+200*i) self.browser.execute_script(js) sleep(0.1) js = "var q=document.documentElement.scrollTop=100000" self.browser.execute_script(js) sleep(0.2) # 爬取天猫商品数据 def crawl_good_data(self): # 点击“淘宝网首页” try: self.browser.find_element_by_xpath("/html/body/div[1]/div/ul[2]/li[1]/div").click() except: self.browser.get("https://www.taobao.com/") self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="q"]'))).send_keys("羽毛球") self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="q"]'))).send_keys(Keys.ENTER) page_total = self.search_toal_page() print("总共页数" + page_total) # 遍历所有页数 for page in range(2,int(page_total)): # 等待该页面input输入框加载完毕 input = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div#mainsrp-pager > div.m-page.g-clearfix > div.wraper > div.inner.clearfix > div.form > input.input.J_Input'))) # 获取当前页 now_page = int(input.get_attribute('value')) -1 print("当前页数" + str(now_page) + ",总共页数" + page_total) # 获取本页面源代码 html = self.browser.page_source # pq模块解析网页源代码 doc = pq(html) # 存储商品数据 good_items = doc('div#mainsrp-itemlist > div.m-itemlist > div.grid.g-clearfix > div.items').items() # 遍历该页的所有商品 for item in good_items: _item = item.find('.J_ClickStat'); good_title = _item.text() good_status = _item.attr('trace-num') good_price = _item.attr('trace-price') good_url = _item.attr('href') self.es.append_data(good_title, good_status, good_price, good_url) # 精髓之处，大部分人被检测为机器人就是因为进一步模拟人工操作 # 模拟人工向下浏览商品，即进行模拟下滑操作，防止被识别出是机器人 self.swipe_down(2) # 翻页，下一页 self.next_page(page) try: # 等待滑动验证码出现,超时时间为5秒，每0.5秒检查一次 # 大部分情况不会出现滑动验证码，所以如果有需要可以注释掉下面的代码 # sleep(5) WebDriverWait(self.browser, 5, 0.5).until(EC.presence_of_element_located((By.ID, "nc_1_n1z"))) #等待滑动拖动控件出现 swipe_button = self.browser.find_element_by_id('nc_1_n1z') #获取滑动拖动控件 #模拟拽托 action = ActionChains(self.browser) # 实例化一个action对象 action.click_and_hold(swipe_button).perform() # perform()用来执行ActionChains中存储的行为 action.reset_actions() action.move_by_offset(580, 0).perform() # 移动滑块 except Exception as e: print ('get button failed: ', e) if __name__ == "__main__": chromedriver_path = "/opt/google/chrome/chromedriver" username = "你的淘宝帐号" password = "你的密码" a = taobao_infos() a.login() a.crawl_good_data()
3：数据写入es
from elasticsearch import Elasticsearch request_body = { "mappings": { "properties": { "_title": { "type": "text" }, "_status": { "type": "keyword" }, "_price": { "type": "keyword" }, "_url": { "type": "keyword" } } } } class es_storage: def __init__(self, host): self.es = Elasticsearch(host); self.index = 'student'; # 索引不存在，则创建 if self.es.indices.exists(self.index) == False: self.es.indices.create(index=self.index, body=request_body) def append_data(self,_title ,_status ,_price ,_url): self.es.index(index=self.index, body={"_title": _title, "_status": _status, "_price":_price, "_url":_url}) def del_index(self): self.es.indices.delete(index=self.index) def search_index(self): dsl = {"query":{"match_all":{}}} res = self.es.search(index=self.index, body=dsl) print(res) if __name__ == "__main__": e = es_storage(['http://127.0.0.1:9200']) e.search_index()
4：注意
登入可以不会通过，须要手动滑动，手输短信验证码； 其它功能放心食用。
