appium+fiddler——douyin小爬虫
//保存到本地添加开始 //这是抖音的地址||"v1-dy.ixigua.com"||"v3-dy.ixigua.com"||"v6-dy.ixigua.com"||"v9-dy.ixigua.com"|| if (oSession.fullUrl.Contains("v1-dy.ixigua.com")|| oSession.fullUrl.Contains("v3-dy.ixigua.com")|| oSession.fullUrl.Contains("v6-dy.ixigua.com")|| oSession.fullUrl.Contains("v9-dy.ixigua.com")){ var fso; var file; fso = new ActiveXObject("Scripting.FileSystemObject"); //文件保存路径,可自定义 file = fso.OpenTextFile("H:\\Request.txt",8 ,true); //file.writeLine("Request-url:" + oSession.url); file.writeLine("http://"+oSession.url) //file.writeLine("Request-host:" + oSession.host); //file.writeLine("Request-header:" + "\n" + oSession.oRequest.headers); //file.writeLine("Request-body:" + oSession.GetRequestBodyAsString()); //file.writeLine("\n"); file.close(); } //保存到本地添加结束
from appium import webdriver from time import sleep #以下代码可以操控手机app(抖音) class Action(): def __init__(self): #初始化配置,设置Desired Capabilities参数 self.desired_caps = { "platformName": "Android", "deviceName": "Mi_Note_3", "appPackage": "com.ss.android.ugc.aweme", "appActivity": ".main.MainActivity" } #指定Appium Server self.server = 'http://localhost:4723/wd/hub' #新建一个Session self.driver = webdriver.Remote(self.server, self.desired_caps) #设置滑动初始坐标和滑动距离 self.start_x = 500 self.start_y = 1500 self.distance = 1300 def comments(self): sleep(3) #app开启之后点击一次屏幕,确保页面的展示 self.driver.tap([(500, 1200)], 500) def scroll(self): #无限滑动 while True: #模拟滑动 self.driver.swipe(self.start_x, self.start_y, self.start_x, self.start_y - self.distance) #设置延时等待 sleep(5) def main(self): self.comments() self.scroll() if __name__ == '__main__': action = Action() action.main()
# _*_ coding: utf-8 _*_ import requests import sys headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36" } #去重方法 def distinct_data(): #读取txt中文档的url列表 datalist_blank=[] pathtxt='H:/Request.txt' with open(pathtxt) as f: f_data_list=f.readlines()#d得到的是一个list类型 for a in f_data_list: datalist_blank.append(a.strip())#去掉\n strip去掉头尾默认空格或换行符 data_dict={} for data in datalist_blank: #url中以/为切分,在以m为切分 ##把m后面的值放进字典key的位置,利用字典特性去重 if int(data.split('/').index('m'))==4 :#此处为v6开头的url #print(data,44,data.split('/')[5]) data_key1=data.split("/")[5] data_dict[data_key1]=data elif int(data.split('/').index('m'))==6: #此处为v1或者v3或者v9开头的url data_key2=data.split("/")[7] data_dict[data_key2] =data #print(len(data_dict),data_dict) data_new=[] for x,y in data_dict.items(): data_new.append(y) return data_new def responsedouyin(): data_url=distinct_data() #使用request获取视频url的内容 #stream=True作用是推迟下载响应体直到访问Response.content属性 #将视频写入文件夹 num = 1 for url in data_url: res = requests.get(url,stream=True,headers=headers) #res = requests.get(url=url, stream=True, headers=headers) #定义视频存放的路径 pathinfo = 'H:/douyin-video/%d.mp4' % num #%d 用于整数输出 %s用于字符串输出 #实现下载进度条显示,这一步需要得到总视频大小 total_size = int(res.headers['Content-Length']) #设置流的起始值为0 temp_size = 0 if res.status_code == 200: with open(pathinfo, 'wb') as file: #file.write(res.content) #print(pathinfo + '下载完成啦啦啦啦啦') num += 1 #当流下载时,下面是优先推荐的获取内容方式,iter_content()函数就是得到文件的内容,指定chunk_size=1024,大小可以自己设置哟,设置的意思就是下载一点流写一点流到磁盘中 for chunk in res.iter_content(chunk_size=1024): if chunk: temp_size += len(chunk) file.write(chunk) file.flush() #刷新缓存 #############下载进度条部分start############### done = int(50 * temp_size / total_size) #print('百分比:',done) sys.stdout.write("\r[%s%s] %d % %" % ('█' * done, ' ' * (50 - done), 100 * temp_size / total_size)+" 下载信息:"+pathinfo + "下载完成啦啦啦啦啦") sys.stdout.flush()#刷新缓存 #############下载进度条部分end############### print('\n')#每一条打印在屏幕上换行输出 if __name__ == '__main__': responsedouyin()