python 从url中获取数据
import requests #通过url获取数据 def get_page(url): #requests.get 自带 json.load page = requests.get(url) page = page.content #将bytes转换成字符串 page = page.decode('utf-8') return page # print(get_page('https://www.duitang.com/napi/blog/list/by_search/?kw=%E6%A0%A1%E8%8A%B1&start=0&limit=1000'))
r= get_page('https://adventofcode.com/2020/day/2/input')
Puzzle inputs differ by user. Please log in to get your puzzle input.\n'
如何模拟登陆呢?
封装成一个类:
import sys import io from urllib import request class Data(): sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') # 改变标准输出的默认编码 def __init__(self): #浏览器登录后得到的cookie,也就是刚才复制的字符串 self.cookie_str = r'_ga=GA1.2.497555532.1608957411; _gid=GA1.2.648427415.1608957411; session=53616c7465645f5fec6238fdbf7a245cd4cc6ecfc3c8e7cbf293b6acc0ab9320cb23950d9c67b8346f3812daa8a7b417' def getData(self, url = r'https://adventofcode.com/2020/day/2/input'): # 登录后才能访问的网站 req = request.Request(url) # 设置cookie req.add_header('cookie', self.cookie_str) # 设置请求头 req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36') resp = request.urlopen(req) return(resp.read().decode('utf-8'))
这样子调用:
from ChristmasGame.Data import * Data = Data() r = Data.getData() print(r)