xpath
xpath介绍
1.html中选择标签，可以使用的通用方法
-css选择
-xpath选择
1.1XPath即为XML路径语言（XML Path Language），它是一种用来确定XML文档中某部分位置的语言
1.2语法介绍
nodename 选取此节点的所有子节点
/ 从根节点选取 /body/div
// 从匹配选择的当前节点选择文档中的节点，而不考虑它们的位置 //div
. 选取当前节点
.. 选取当前节点的父节点
@ 选取属性
1.3便利的使用方法
在浏览器的抓包工具中，找到自己需要的数据的标签，在标签上右键，点击在Copy中的Copy XPath，获取了获取标签需要的xpath路径
xpath的使用
doc = '''
<html>
<head>
<base href='http://example.com/' />
<title>Example website</title>
</head>
<body>
<div id='images'>
<a href='image1.html' id='id_a'>Name: My image 1 <br/><img src='image1_thumb.jpg' /></a>
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
<a href='image5.html' class='li li-item' name='items'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
<a href='image6.html' name='items'><span><h5>test</h5></span>Name: My image 6 <br /><img src='image6_thumb.jpg' /></a>
</div>
</body>
</html>
'''
from lxml import etree
html = etree.HTML(doc)
a = html.xpath('//*')
a = html.xpath('//head')
a = html.xpath('//div/a')
a = html.xpath('//body/a')
a = html.xpath('//body//a')
a = html.xpath('//body//a[@href="image1.html"]/..')
a = html.xpath('//body//a[1]/..')
a = html.xpath('//body//a[1]/parent::*')
a = html.xpath('//body//a[1]/parent::div')
a = html.xpath('//body//a[@href="image1.html"]')
a = html.xpath('//body//a[@href="image1.html"]/text()')
a = html.xpath('//body//a/@href')
a = html.xpath('//body//a/@id')
a = html.xpath('//body//a[1]/@id')
a = html.xpath('//body//a[@class="li"]')
a = html.xpath('//body//a[@name="items"]')
a = html.xpath('//body//a[contains(@class,"li")]')
a = html.xpath('//body//a[contains(@class,"li")]/text()')
a = html.xpath('//body//a[contains(@class,"li") or @name="items"]')
a = html.xpath('//body//a[contains(@class,"li") and @name="items"]/text()')
a = html.xpath('//a[2]/text()')
a = html.xpath('//a[3]/@href')
a = html.xpath('//a[last()]/@href')
a = html.xpath('//a[position()<3]/@href')
a = html.xpath('//a[last()-2]/@href')
a = html.xpath('//a/ancestor::*')
a = html.xpath('//a/ancestor::div')
a = html.xpath('//a[1]/attribute::*')
a = html.xpath('//a[1]/attribute::href')
a = html.xpath('//a[1]/child::*')
a = html.xpath('//a[6]/descendant::*')
a = html.xpath('//a[1]/following::*')
a = html.xpath('//a[1]/following::*[1]/@href')
a = html.xpath('//a[1]/following-sibling::*')
a = html.xpath('//a[1]/following-sibling::a')
a = html.xpath('//a[1]/following-sibling::*[2]')
a = html.xpath('//a[1]/following-sibling::*[2]/@href')
print(a)
selenium动作链
1.通过代码实现按住，滑动的效果，如:滑动验证码
2.两种形式
2.1形式1
actions=ActionChains(bro)
actions.drag_and_drop(sourse,target)
actions.perform()
2.2形式2
ActionChains(bro).click_and_hold(sourse).perform()
distance=target.location['x']-sourse.location['x']
track=0
while track < distance:
ActionChains(bro).move_by_offset(xoffset=2,yoffset=0).perform()
track+=2
3.例子
3.1
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
bro = webdriver.Chrome(executable_path='./chromedriver.exe')
bro.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
bro.implicitly_wait(10)
try:
bro.switch_to.frame('iframeResult')
sourse = bro.find_element(by=By.ID, value='draggable')
target = bro.find_element(by=By.ID, value='droppable')
actions = ActionChains(bro)
actions.drag_and_drop(sourse, target)
actions.perform()
except Exception as e:
print(e)
3.2
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
bro = webdriver.Chrome(executable_path='./chromedriver.exe')
bro.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
bro.implicitly_wait(10)
try:
bro.switch_to.frame('iframeResult')
sourse = bro.find_element(by=By.ID, value='draggable')
target = bro.find_element(by=By.ID, value='droppable')
ActionChains(bro).click_and_hold(sourse).perform()
distance = target.location['x'] - sourse.location['x']
track = 0
while track < distance:
ActionChains(bro).move_by_offset(xoffset=5, yoffset=0).perform()
track += 5
except Exception as e:
print(e)
自动登录12306
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time
options = Options()
options.add_argument("--disable-blink-features=AutomationControlled")
bro = webdriver.Chrome(executable_path='./chromedriver.exe', options=options)
bro.get('https://kyfw.12306.cn/otn/resources/login.html')
bro.maximize_window()
bro.implicitly_wait(10)
try:
username = bro.find_element(by=By.ID, value='J-userName')
username.send_keys('')
password = bro.find_element(by=By.ID, value='J-password')
password.send_keys('')
time.sleep(2)
btn = bro.find_element(by=By.ID, value='J-login')
btn.click()
span = bro.find_element(by=By.ID, value='nc_1_n1z')
ActionChains(bro).click_and_hold(span).perform()
ActionChains(bro).move_by_offset(xoffset=300, yoffset=0).perform()
time.sleep(10)
except Exception as e:
print(e)
finally:
bro.close()
打码平台自动登录
1.在超级鹰中下载python语言Demo，将下载的py文件复制到项目中
2.使用
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from chaojiying import ChaojiyingClient
from PIL import Image
bro = webdriver.Chrome(executable_path='./chromedriver.exe')
bro.get('http://www.chaojiying.com/apiuser/login/')
bro.implicitly_wait(10)
bro.maximize_window()
try:
username = bro.find_element(by=By.XPATH, value='/html/body/div[3]/div/div[3]/div[1]/form/p[1]/input')
password = bro.find_element(by=By.XPATH, value='/html/body/div[3]/div/div[3]/div[1]/form/p[2]/input')
code = bro.find_element(by=By.XPATH, value='/html/body/div[3]/div/div[3]/div[1]/form/p[3]/input')
btn = bro.find_element(by=By.XPATH, value='/html/body/div[3]/div/div[3]/div[1]/form/p[4]/input')
username.send_keys('')
password.send_keys('')
bro.save_screenshot('main.png')
img = bro.find_element(By.XPATH, '/html/body/div[3]/div/div[3]/div[1]/form/div/img')
location = img.location
size = img.size
print(location)
print(size)
img_tu = (
int(location['x']), int(location['y']), int(location['x'] + size['width']), int(location['y'] + size['height']))
img = Image.open('./main.png')
fram = img.crop(img_tu)
fram.save('code.png')
chaojiying = ChaojiyingClient('', '', '')
im = open('code.png', 'rb').read()
print(chaojiying.PostPic(im, 1902))
res_code = chaojiying.PostPic(im, 1902)['pic_str']
code.send_keys(res_code)
time.sleep(2)
btn.click()
except Exception as e:
print(e)
finally:
bro.close()
使用selenium爬取京东商品信息
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.common.keys import Keys
def get_goods(driver):
try:
goods = driver.find_elements(by=By.CLASS_NAME, value='gl-item')
for good in goods:
name = good.find_element(by=By.CSS_SELECTOR, value='.p-name em').text
price = good.find_element(by=By.CSS_SELECTOR, value='.p-price i').text
commit = good.find_element(by=By.CSS_SELECTOR, value='.p-commit a').text
url = good.find_element(by=By.CSS_SELECTOR, value='.p-name a').get_attribute('href')
img = good.find_element(by=By.CSS_SELECTOR, value='.p-img img').get_attribute('src')
if not img:
img = 'https://' + good.find_element(by=By.CSS_SELECTOR, value='.p-img img').get_attribute(
'data-lazy-img')
print('''
商品名字：%s
商品价格：%s
商品链接：%s
商品图片：%s
商品评论：%s
''' % (name, price, url, img, commit))
button = driver.find_element(by=By.PARTIAL_LINK_TEXT, value='下一页')
button.click()
time.sleep(1)
get_goods(driver)
except Exception as e:
print(e)
def spider(url, keyword):
driver = webdriver.Chrome(executable_path='./chromedriver.exe')
driver.get(url)
driver.implicitly_wait(10)
try:
input_tag = driver.find_element(by=By.ID, value='key')
input_tag.send_keys(keyword)
input_tag.send_keys(Keys.ENTER)
get_goods(driver)
finally:
driver.close()
if __name__ == '__main__':
spider('https://www.jd.com/', keyword='精品内衣')
scrapy介绍
1.专业做爬虫的，可以使用框架(django:web)scrapy:爬虫框架
-做爬虫的框架，封装了做爬虫的东西，只需要在固定的位置写固定的代码即可
2.scrapy 号称爬虫界的djagno
-django 大而全，做web相关的它都用
-scrapy 大而全，做爬虫的，它都用
2.1介绍
Scrapy一个开源和协作的框架，其最初是为了页面抓取 (更确切来说, 网络抓取 )所设计的，使用它可以以快速、简单、可扩展的方式从网站中提取所需的数据。但目前Scrapy的用途十分广泛，可用于如数据挖掘、监测和自动化测试等领域，也可以应用在获取API所返回的数据或者通用的网络爬虫
3.安装 scrapy
3.1mac，linux：
pip3 install scrapy
3.2win：看人品
pip3 install scrapy
-人品不好：
1、pip3 install wheel
3、pip3 install lxml
4、pip3 install pyopenssl
5、下载并安装pywin32：https://sourceforge.net/projects/pywin32/files/pywin32/
6、下载twisted的wheel文件：http://www.lfd.uci.edu/~gohlke/pythonlibs/
7、执行pip3 install 下载目录\Twisted-17.9.0-cp36-cp36m-win_amd64.whl
8、pip3 install scrapy
4.使用
4.1释放出scrapy 可执行文件
使用这个创建爬虫项目 ---》django-admin创建django项目
4.2创建爬虫项目
scrapy startproject myfirstscrapy
4.3创建爬虫 [django创建app]
scrapy genspider cnblogs www.cnblogs.com
4.4启动爬虫
scrapy crawl cnblogs
scrapy crawl cnblogs --nolog
4.5pycharm中运行
from scrapy.cmdline import execute
execute(['scrapy', 'crawl', 'cnblogs', '--nolog'])
【推荐】国内首个AI IDE，深度理解中文开发场景，立即下载体验Trae
【推荐】编程新体验，更懂你的AI，立即体验豆包MarsCode编程助手
【推荐】抖音旗下AI助手豆包，你的智能百科全书，全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell：AI 加持，快人一步
· 震惊！C++程序真的从main开始吗？99%的程序员都答错了
· 别再用vector＜bool＞了！Google高级工程师：这可能是STL最大的设计失误
· 单元测试从入门到精通
· 【硬核科普】Trae如何「偷看」你的代码？零基础破解AI编程运行原理
· 上周热点回顾（3.3-3.9）