Python全栈开发武沛齐day07模块
day07 模块
1.知识回顾
• 模块的分类
- 自定义模块
- 内置
- 第三方模块
• 自定义模块
– 关于拆分
crm - utils
- encrypt.py
- db.py
- message.py
- app.py
– 文件夹和文件(扩展)
py文件 -> 模块
文件夹 -> 包
py2的包:内部必须有一个 init.py
py3的包:无限制
– 导入模块
• 去哪里导入?
import sys
sys.path
当前运行脚本所在目录
python的安装目录
python的安装目录/site-packages
注意:pycharm自动将项目目录加入sys.path【应该忽略】
• 怎么导入?
import ???
from xxxx.xxxx import ???
from xxxx.xxxx import *
from xxxx.xxxx import xxx
from xxxx.xxxx import xxx as ???
from xxxx.xxxx import xxx,xx,x
– 主文件+主函数
def func():
pass
func()
def func():
pass
if name == "main":
func()
当一个py文件被运行时,__name__就是"main"
当一个py文件被导入时,__name__就是模块的名字,例如:utils.xxx.info
• 内置模块
– random
– hashlib
– json
– time/datetime
– os
2.内置模块
2.1 os
• 路径拼接
import os
path = os.path.join("xxx","xxxx","xxxxx","xxxx.txt")
• 上级目录
import os
path = os.path.dirname(".....")
• 绝对路径
import os
os.path.abspath(".....")
os.path.abspath(file)
import os
base_dir = os.path.dirname(os.path.abspath(file))
• 判断路径是否存在
import os
v1 = os.path.exists("路径")
print(v1) # True/False
import os
user = input("用户名:")
file_path = os.path.join("files", "db.txt")
判断文件夹是否存在?
if os.path.exists(os.path.dirname(file_path)):
with open(file_path, mode='a', encoding='utf-8') as f:
f.write(f"{user}\n")
else:
print("路径不存在")
import os
file_path = os.path.join("files", 'db.txt')
if os.path.exists(file_path):
with open(file_path, mode='r', encoding='utf-8') as f:
data = f.read()
print(data)
else:
print("文件不存在")
• 创建文件夹(文件夹不存在)
import os
os.makedirs("xxx/xxxx/xxxx")
import os
user = input("用户名:")
file_path = os.path.join("files", "db.txt")
folder_path = os.path.dirname(file_path)
if not os.path.exists(folder_path):
os.makedirs(folder_path)
with open(file_path, mode='a', encoding='utf-8') as f:
f.write(f"{user}\n")
import os.path
import requests
from datetime import datetime
pip install requests
url_list = [
"https://www3.autoimg.cn/newsdfs/g30/M05/3C/5E/400x300_0_autohomecar__ChxknGRTwLSAP9MuAAA6hd6ZZDY038.jpg",
"https://www3.autoimg.cn/cubetopic/g27/M04/43/F9/400x300_0_autohomecar__ChxkmWRTWkaACfQbAADCg_L8aQM773.jpg"
]
for url in url_list:
res = requests.get(url=url)
# 图片写入本地文件
date_string = datetime.now().strftime("%Y-%m-%d")
name = url.split("__")[-1]
if not os.path.exists(date_string):
os.makedirs(date_string)
file_path = os.path.join(date_string, name)
with open(file_path, mode='wb') as f:
f.write(res.content)
• 判断文件 或 文件夹
os.path.isdir(...)
• 删除文件和文件夹
import os
os.remove("文件路径") # 删除文件
import shutil
shutil.rmtree("文件夹的路径") # 删除文件夹
• 查看目录下的文件或文件夹【一级目录】
import os
name_list = os.listdir("路径")
import os.path
folder_path = os.path.join("2023-05-05")
name_list = os.listdir(folder_path)
for name in name_list:
inner_path = os.path.join(folder_path,name)
if os.path.isdir(inner_path):
print(os.listdir(inner_path))
else:
print(inner_path)
• 查看目录下的文件或文件夹【多级目录】
import
os.walk("路径")
import os
folder_path = os.path.join("2023-05-05")
for base_dir, folder_list, file_list in os.walk(folder_path):
for name in file_list:
file_path = os.path.join(base_dir,name)
print(file_path)
import os
folder_path = os.path.join(r"E:\EvVideo\其他视频")
for base_dir, folder_list, file_list in os.walk(folder_path):
for name in file_list:
if not name.endswith(".mp4"):
continue
file_path = os.path.join(base_dir, name)
print(file_path)
案例:用户注册
- 输入:用户名+密码(md5加密)
- 写入文件
2023-11-11
12-09.txt
12-10.txt
2023-11-12
12-09.txt
12-10.txt
import hashlib
import os.path
from datetime import datetime
MD5_SALT = "asdfasdkfojlkjao9urpqoiwj;lkafjsdf"
def md5(data_str):
obj = hashlib.md5(MD5_SALT.encode('utf-8'))
obj.update(data_str.encode('utf-8'))
return obj.hexdigest()
def gen_file_path():
ctime = datetime.now()
date_string = ctime.strftime("%Y-%m-%d")
time_string = ctime.strftime("%H-%M")
# 3.1 文件夹处理
if not os.path.exists(date_string):
os.makedirs(date_string)
# 3.2 文件路径
file_path = os.path.join(date_string, f"{time_string}.txt")
return file_path
def run():
# 1.输入用户名和密码
user = input("用户名:")
pwd = input("密码:")
md5_pwd = md5(pwd)
# 2.拼接
line = f"{user},{md5_pwd}\n"
# 3.创建路径+写入
file_path = gen_file_path()
with open(file_path, mode='a', encoding='utf-8') as f:
f.write(line)
if name == 'main':
run()
1分钟就会生成1个文件,
2.2 sys
import sys
1.导入模块时,都会去哪里找
sys.path
2.sys.argv,运行脚本时传入的参数
sys.argv
这东西有什么用?
• IT相关同学:shell脚本
• 非IT相关的同学
写程序去实现下载某个视频的一个需求。
- 方式1:input来实现
url = input("请输入下载网址:") - 方式2:argv
import sys
url = sys.argv[1]
print("下载:", url)
2.3 打包代码
pip3.9 install pyinstaller
2.3.1 多文件打包
pyinstaller -D client.py
2.3.2 单文件打包(喜欢)
pyinstaller -F client.py
如果你的程序中会涉及到文件的操作,文件放在项目的相对目录。【BUG】
当运行 client.exe 文件时,内部有很多的代码,会放在电脑的临时目录。
os.path.abspath(file)
C:\Users\ADMINI~1\AppData\Local\Temp_MEI77322\client.py
• 如果使用的【绝对路径】+【file】 【错误】
import os
def run():
base_dir = os.path.dirname(os.path.abspath(file))
db_file_path = os.path.join(base_dir, "db.txt")
with open(db_file_path, mode='r', encoding='utf-8') as f:
print(f.read())
user = input("用户名:")
pwd = input("密码:")
line = f"{user},{pwd}"
print(line)
input("回车继续")
if name == 'main':
run()
双击的话,一闪而过
把文件拖到cmd里就能如上面看到报错
• 如果使用的【绝对路径】+【sys.argv】 【正确】
import os
import sys
def run():
base_dir = os.path.dirname(sys.argv[0])
db_file_path = os.path.join(base_dir, "db.txt")
with open(db_file_path, mode='r', encoding='utf-8') as f:
print(f.read())
user = input("用户名:")
pwd = input("密码:")
line = f"{user},{pwd}"
print(line)
input("回车继续")
if name == 'main':
run()
• 相对路径 【正确】
def run():
with open("db.txt", mode='r', encoding='utf-8') as f:
print(f.read())
user = input("用户名:")
pwd = input("密码:")
line = f"{user},{pwd}"
print(line)
input("回车继续")
if name == 'main':
run()
小结
后期如果你的项目需要读取文件 + 文件夹 等需求,推荐使用绝对路径。
• 开发阶段:代码运行
• 打包阶段:exe运行
import os
import sys
BASE_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
def run():
with open(os.path.join(BASE_DIR,'db.txt'), mode='r', encoding='utf-8') as f:
print(f.read())
user = input("用户名:")
pwd = input("密码:")
line = f"{user},{pwd}"
print(line)
input("回车继续")
if name == 'main':
run()
2.3 configparser
• 用txt文件【配置文件操作】
xxx=123
xxxx=123
• 专门用于对于 .ini 格式的文件【一般用于项目的配置文件】 *
[server]
v1=123
v2=456
[client]
v9=111
• 使用JSON格式 .json格式 【一般用于项目的配置文件】
{
"count":123,
"info":[11,22,33]
}
• 专门使用xml格式文件【一般用于项目的配置文件】
• Python的模块 settings.py 【无法再打包后进行修改生效】
COUNT = 123
INFO = 999
2.3.1 基本操作
import configparser
1.打开并读取文件
obj = configparser.ConfigParser()
obj.read("my.ini", encoding='utf-8')
2.读取节点
v1 = obj.sections()
print(v1)
[‘server’,’client’]
3.键值对
v2 = obj.items("server")
for k,v in v2:
print(k,v)
v2
4.键值
v3 = obj.get("server",'v3')
print(v3)
中国联通
5.包含
v3 = obj.has_section("server")
print(v3)
True
6.添加
obj.add_section("group")
obj.set("group", 'name', "武沛齐")
obj.set("group", 'age', "19")
with open("my.ini", mode='w', encoding='utf-8') as f:
obj.write(f)
7.删除
obj.remove_section("group")
with open("my.ini", mode='w', encoding='utf-8') as f:
obj.write(f)
8.删除
obj.remove_option("server", "v2")
with open("my.ini", mode='w', encoding='utf-8') as f:
obj.write(f)
9.修改
obj.set("server", "v1", "999")
with open("my.ini", mode='w', encoding='utf-8') as f:
obj.write(f)
2.3.2 应用案例
1.价格监测平台
通过爬虫的程序,一直请求网页,查看他得价格。一旦低于我的预期,我就进行消息的通知。
监测过程中需求配置:
• 间隔时间:30
• 企业微信:群 + 群机器人 + 网址发送
import configparser
import os
import sys
BASE_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
SETTINGS_PATH = os.path.join(BASE_DIR, 'my.ini')
def run():
# 1.加载配置文件
# 1.1 找到文件
if not os.path.exists(SETTINGS_PATH):
print("配置文件不存在")
input("")
return
# 2.2 读取
config_dict = {}
config = configparser.ConfigParser()
config.read(SETTINGS_PATH, encoding='utf-8')
for k, v in config.items("server"):
config_dict[k] = v
# 2.根据配置编写代码
print(config_dict)
input("功能实现...")
input("回车继续")
if name == 'main':
run()
打包后要把my.ini放在dist下,和client.exe放在一起,才能正常运行,不放一起的话,会提示配置文件不存在,然后退出
import configparser
import os
import sys
import time
import requests
BASE_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
SETTINGS_PATH = os.path.join(BASE_DIR, 'my.ini')
def fetch_new_info():
return True
def run():
# 1.加载配置文件
# 1.1 找到文件
if not os.path.exists(SETTINGS_PATH):
print("配置文件不存在")
input("")
return
# 2.2 读取
config_dict = {}
config = configparser.ConfigParser()
config.read(SETTINGS_PATH, encoding='utf-8')
for k, v in config.items("server"):
config_dict[k] = v
# 2.根据配置编写代码
print(config_dict)
interval = int(config_dict["interval"])
while True:
status = fetch_new_info()
if status:
break
time.sleep(interval)
# 3.消息通知 [企业微信]+[群]+[机器人]
# https://www.zhihu.com/question/395840381/answer/2278274881
notify_url = config_dict['notify']
input("功能实现...")
input("回车继续")
if name == 'main':
run()
import configparser
import os
import sys
import time
import requests
BASE_DIR = os.path.dirname(os.path.realpath(sys.argv[0]))
SETTINGS_PATH = os.path.join(BASE_DIR, 'my.ini')
CONFIG_DICT = None
def fetch_new_info():
return True
def process():
print(CONFIG_DICT)
interval = int(CONFIG_DICT["interval"])
while True:
status = fetch_new_info()
if status:
break
time.sleep(interval)
def notify():
# https://www.zhihu.com/question/395840381/answer/2278274881
notify_url = CONFIG_DICT['notify']
def load_config():
# 1.1 找到文件
if not os.path.exists(SETTINGS_PATH):
return False, "配置文件不存在"
# 2.2 读取
config_dict = {}
config = configparser.ConfigParser()
config.read(SETTINGS_PATH, encoding='utf-8')
for k, v in config.items("server"):
config_dict[k] = v
return True, config_dict
def run():
# 1.加载配置文件
status, config_dict = load_config()
if not status:
print(config_dict)
input("")
return
global CONFIG_DICT
CONFIG_DICT = config_dict
# 2.根据配置编写代码
process()
# 3.消息通知 [企业微信]+[群]+[机器人]
notify()
input("功能实现...")
input("回车继续")
if name == 'main':
run()
2.4 XML
一种特殊的格式,用于表示和存储数据。
{
"name":"武沛齐",
"age":18
}
text = """
"""
from xml.etree import ElementTree
root = ElementTree.XML(text)
1.第一个
node = root.find("country")
print(node)
print(node.tag)
print(node.attrib)
2.找到多个
node_list = root.findall("country")
for node in node_list:
print(node.tag, node.attrib)
3.继续向下找
node_list = root.findall("country")
for node in node_list:
# print(node.tag, node.attrib)
# res = node.getchildren()
# print(res)
# rank = node.find("rank")
# print(rank.text, rank.attrib)
for child in node:
print(child.tag, child.text, child.attrib)
案例:腾讯接口API
from xml.etree import ElementTree
text = """
"""
data_dict = {}
root = ElementTree.XML(text)
for node in root:
data_dict[node.tag] = node.text
print(data_dict)
2.5 re模块
• 正则表达式,提取or校验你的数据。
• re模块Python中的内置模块,用于和正则表达式搭配。
2.5.1 正则表达式
text = "楼主太牛逼了,在线想要 s42662578@qq.com 和 xxxxx@live.com谢谢楼主,手机号也可15131255789,搞起来呀"
需求:提取手机号 \d{11}
需求:提取邮箱 \w+@\w+.\w+
text = "楼主太牛逼了,在15131255781线想要 s42662578@qq.com 和 xxxxx@live.com谢谢楼主,手机号也可15131255789,搞起来呀"
import re
ret = re.findall(r"\d{11}", text)
print(ret)
text = "楼主太牛逼了,在15131255781线想要 s42662578@qq.com 和 xxxxx@live.com谢谢楼主,手机号也可15131255789,搞起来呀"
import re
ret = re.findall(r"\w+@\w+.\w+", text, re.ASCII)
print(ret)
email = input("邮箱:")
import re
ret = re.match(r"^\w+@\w+.\w+$", email)
if ret:
print("格式合法")
else:
print("格式错误")
1.字符相关
• 固定文本
import re
text = "你好wupeiqi,阿斯顿发wupeiqasd 阿士大夫能接受的wupeiqiff"
data_list = re.findall(r"wupeiqi", text)
print(data_list) # ["wupeiqi", "wupeiqi"]
• 含有特定字符串
import re
text = "你好wupeiqi,阿斯顿发wupeiqasd 阿士大夫能接受的wupeiqbiff"
wupeiqa wupeiqb wupeiqi
data_list = re.findall(r"wupeiq[abi]", text)
print(data_list) # ["wupeiqi", "wupeiqa","wupeiqb"]
• 范围 [a-z] [0-9]
import re
text = "你好twupeiqi,阿斯顿发wupetiqasd 阿士大夫能接受的wutpeiqbff"
data_list = re.findall(r"t[a-z]", text)
print(data_list) # ['tw', 'ti', 'tp']
import re
text = "你好twupeiqi,阿斯顿发wupet2iqasd 阿士大夫能接受的wut1peiqbff"
data_list = re.findall(r"t[0-9]", text)
print(data_list)
import re
text = "你好twupeiqi,阿斯顿发wupet2iqasd 阿士大夫能接受的wut1peiqbff"
data_list = re.findall(r"t[0-9][a-z]", text)
print(data_list)
• \d 数字
import re
text = "你好t11wupeiqi,阿斯顿发wupet22iqasd 阿士大夫能接受的wut8peiqbff"
data_list = re.findall(r"t\d", text)
print(data_list) # ['t1', 't2', 't8']
import re
text = "你好t11wupeiqi,阿斯顿发wupet22iqasd 阿士大夫能接受的wut8peiqbff"
data_list = re.findall(r"t\d\d", text)
print(data_list) # ['t11', 't22']
• \w,字母、数字、下划线(汉字)
text = "你t好t11wupeiqi,阿斯顿发wupet21232123iqasd 阿s顿大夫能接受的wut8peiqbff"
import re
result = re.findall(r"阿\w顿",text)
print(result) # ['阿斯顿', '阿s顿']
text = "你t好t11wupeiqi,阿斯顿发wupet21232123iqasd 阿s顿大夫能接受的wut8peiqbff"
import re
result = re.findall(r"阿\w顿",text,re.ASCII)
print(result) # ['阿s顿']
• . 除换行符以外的任意字符
text = "你t好t11wupeiqi,阿斯顿发wupet21232123iqasd 阿s顿大夫能接受的阿\n顿ut8peiqbff"
import re
result = re.findall(r"阿.顿", text)
print(result) # ['阿斯顿', '阿s顿']
2.数量先关
• {n}
• {n,}
• {n,m}
• *,0次或n次
import re
text = "你好t1wupeiqi,阿斯顿发wupetwdiqasd 阿士大夫能接受的wut18weiqbff"
data_list = re.findall(r"t\d*w", text)
print(data_list) # ['t1w', 'tw', 't18w']
• +,1次或n次
import re
text = "你好t1wupeiqi,阿斯顿发wupetwdiqasd 阿士大夫能接受的wut18weiqbff"
data_list = re.findall(r"t\d+w", text)
print(data_list) # ['t1w', 't18w']
• ?,0次或1次
import re
text = "你好t1wupeiqi,阿斯顿发wupetwdiqasd 阿士大夫能接受的wut18weiqbff"
data_list = re.findall(r"t\d?w", text)
print(data_list) # ['t1w', 'tw']
3.分组
• 在正则中匹配成功,再去提取局部数据。
import re
text = "楼主太牛逼了,在线想要 442662578@qq.com和xxxxx@live.com谢谢楼主,手机号也可15131255799,搞起15131255989来呀"
result = re.findall(r"(151(312\d{5}))", text)
print(result)
[('15131255799', '31255799'), ('15131255989', '31255989')]
import re
text = "我的身份证130449197912038879,郭智的身份之是13044919991203887X阿斯顿发士大夫"
res = re.findall('\d{17}[\dX]', text)
print(res)
import re
text = "我的身份证130449197912038879,郭智的身份之是13044919991203887X阿斯顿发士大夫"
res = re.findall('\d{6}\d{4}\d{2}\d{2}\d{3}[\dX]', text)
print(res)
import re
text = "我的身份证130449197912038879,郭智的身份之是13044919991203887X阿斯顿发士大夫"
res = re.findall('(\d{6}(\d{4})(\d{2})(\d{2})\d{3}[\dX])', text)
print(res)
• 表示或
import re
text = "楼主15131root太牛15131alex逼了,在线想要 442662578@qq.com和xxxxx@live.com谢谢楼主,手机号也可15131255789,搞起来呀"
data_list = re.findall(r"15131(2\d{5}|r\w+太)", text)
print(data_list)
4.起始和结束
import re
text = "130449197912038879"
res = re.findall(r'^\d{17}[\dX]$', text)
print(res)
案例
• 身份证号码
import re
text = "dsf130429191912015219k13042919591219521Xkk"
data_list = re.findall(r"\d{17}[\dX]", text)
print(data_list)
• 手机号
import re
text = "我的手机哈是15133377892,你的手机号是1171123啊?"
data_list = re.findall(r"1[3-9]\d{9}", text)
print(data_list)
• 邮箱
import re
text = "楼主太牛逼了,在线想要 442662578@qq.com和xxxxx@live.com谢谢楼主,手机号也可15131255789,搞起来呀"
email_list = re.findall(r"\w+@\w+.\w+", text, re.ASCII)
print(email_list)
import re
text = "楼主太牛逼了,在线想要 442662578@qq.com和xxxxx@live.com谢谢楼主,手机号也可15131255789,搞起来呀"
email_list = re.findall(r"[a-zA-Z0-9_-]+@[a-zA-Z0-9_-]+.[a-zA-Z0-9_-]+", text)
print(email_list)
2.5.2 re模块
• re.findall,去整个文本中找所有符合正则条件的文本。
import re
text = "我的身份证130449197912038879,郭智的身份之是13044919991203887X阿斯顿发士大夫"
res = re.findall('\d{17}[\dX]', text)
print(res) # ['130449197912038879', '13044919991203887X']
• re.search,去整个文本去匹配,返回匹配成功的第一个
import re
text = "我的身份证130449197912038879,郭智的身份之是13044919991203887X阿斯顿发士大夫"
match_object = re.search('\d{17}[\dX]', text)
print(match_object.group()) # 130449197912038879
• re.match,从开始位置进行匹配,返回匹配成功的第一个
import re
text = "130449197912038879,郭智的身份之是13044919991203887X阿斯顿发士大夫"
match_object = re.match('\d{17}[\dX]', text)
print(match_object)
if match_object:
print(match_object.group()) # 130449197912038879
else:
print("失败")
• re.split
text = "武沛齐,123"
data_list = text.split(",")
print(data_list) # ["武沛齐","123"]
import re
text = "武沛齐,123"
data = re.split(r",",text)
print(data)
import re
text = "武沛齐,123-999"
data = re.split(r"[,-]", text)
print(data)
案例:片段
import re
price = "¥5499"
ret = re.findall(r"¥(\d+)",price)
print(ret)
text = "已有2人评价"
ret = re.findall(r"已有(\d+)人评价",text)
print(ret)
3.第三方模块
优秀开发者,开源出来一个模块,供其他使用。
• 使用者:左边
• 开源者:https://www.bilibili.com/video/BV17541187de/
常见命令:
pip install requests
pip uninstall requests
pip list
pip freeze > requirements.txt
pip install -r requirements.txt
配置pip源:
pip3.9 config set global.index-url https://pypi.douban.com/simple/
pip install ???
2.1 requests模块
基于代码实现发送网络请求。
pip install requests
2.1.1 分析请求 + 实现 + json
Request URL: https://www.zhihu.com/api/v4/comment_v5/articles/545093058/root_comment?order_by=score&limit=20&offset=
Request Method: GET
import json
import requests
res = requests.get(
url="https://www.zhihu.com/api/v4/comment_v5/articles/545093058/root_comment?order_by=score&limit=20&offset="
)
data_dict = json.loads(res.text)
for row in data_dict['data']:
content = row['content']
name = row['author']['name']
print(name, content)
2.1.2 分析请求 + 实现 + json
import requests
import json
res = requests.get(
url="https://movie.douban.com/j/search_subjects?type=movie&tag=豆瓣高分&sort=recommend&page_limit=20&page_start=20",
headers={
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
}
)
data_dict = json.loads(res.text)
for row in data_dict['subjects']:
print(row['title'], row['url'])
2.1.3 数据返回值-JSONP格式
• JSON格式,最容易处理
res = reque...
data_dict = json.loads(res.text)
data_dict = res.json()
• JSONP格式
名字({"rate":"9.0","cover_x":1500,"title":"让子弹飞"})
– 去除元素
名字({"rate":"9.0","cover_x":1500,"title":"让子弹飞"})
{"rate":"9.0","cover_x":1500,"title":"让子弹飞"}
– eval,将一段字符串当做python代码进行编译执行。
def demo(arg):
print(arg)
eval('demo({"rate":"9.0","cover_x":1500,"title":"让子弹飞"})')
import json
import requests
res = requests.get(
url="http://num.10010.com/NumApp/NumberCenter/qryNum?callback=jsonp_queryMoreNums&provinceCode=11&cityCode=110&advancePayLower=0&sortType=1&goodsNet=4&searchCategory=3&qryType=02&channel=B2C&numNet=186&groupKey=53271060&judgeType=1"
)
content = res.text
字符串处理
result = content.strip("jsonp_queryMoreNums(").strip(")")
print(result)
data_dict = json.loads(result)
print(data_dict)
import json
import requests
def jsonp_queryMoreNums(data_dict):
print(data_dict)
res = requests.get(
url="http://num.10010.com/NumApp/NumberCenter/qryNum?callback=jsonp_queryMoreNums&provinceCode=11&cityCode=110&advancePayLower=0&sortType=1&goodsNet=4&searchCategory=3&qryType=02&channel=B2C&numNet=186&groupKey=53271060&judgeType=1"
)
content = res.text
字符串处理
eval(content)
2.1.4 HTML格式
你看到网站的所有的内容,本质上都是由HTML标签给他包裹。
res = requests.get(
url="https://www.autohome.com.cn/news/"
)
res.encoding = 'gb2312'
print(res.text)
1.将文本交给BeautifulSoup进行处理
soup = BeautifulSoup(res.text, features="html.parser")
2.使用对象就是整个文本根据特征寻找标签
tag = soup.find(name="div", attrs={"id": "auto-channel-lazyload-article"})
3.继续往下找
li_list = tag.find_all(name="li")
for node in li_list:
h3_tag = node.find(name="h3")
if not h3_tag:
continue
p_tag = node.find(name="p")
img_tag = node.find(name='img')
print(h3_tag.text)
print(p_tag.text)
print(img_tag.attrs['src'])
print('------------------')
import requests
from bs4 import BeautifulSoup
res = requests.get(
url="https://www.autohome.com.cn/news/"
)
res.encoding = 'gb2312'
print(res.text)
1.将文本交给BeautifulSoup进行处理
soup = BeautifulSoup(res.text, features="html.parser")
2.使用对象就是整个文本根据特征寻找标签
tag = soup.find(name="ul", attrs={"id": "tagInfo"})
3.每个元素
node_list = tag.find_all(name="li")
for li_node in node_list:
name = li_node.find(name="div", attrs={"class": "editorname"}).text
src = li_node.find(name='img').attrs['src']
src_url = f"https:{src}"
print(name, src_url)
import requests
from bs4 import BeautifulSoup
res = requests.get(
url="https://www.autohome.com.cn/news/"
)
res.encoding = 'gb2312'
print(res.text)
1.将文本交给BeautifulSoup进行处理
soup = BeautifulSoup(res.text, features="html.parser")
2.使用对象就是整个文本根据特征寻找标签
tag = soup.find(name="ul", attrs={"id": "tagInfo"})
3.每个元素
node_list = tag.find_all(name="li")
for li_node in node_list:
name = li_node.find(name="div", attrs={"class": "editorname"}).text
src = li_node.find(name='img').attrs['src']
src_url = f"https:{src}"
print(name, src_url)
# 根据URL去发送请求,下载图片的内容
response = requests.get(url=src_url)
with open(f"{name}.jpg",mode='wb') as f:
f.write(response.content)
案例:联通商品商城
import requests
from bs4 import BeautifulSoup
1.发送网络请求,获取文本数据
res = requests.get(
url="http://s.10010.com/bj/mobile/"
)
print(res.text)
2.解析数据bs4
soup = BeautifulSoup(res.text, features="html.parser")
3.寻找特征+获取内部元素
tag = soup.find(name='div', attrs={'id': "goodsList"})
4.寻找每个商品
li_list = tag.find_all(name='li', attrs={"class": 'goodsLi'})
for li_node in li_list:
title = li_node.find(name='p', attrs={"class": "mobileGoodsName"}).text.strip()
price = li_node.find(name='p', attrs={"class": "evaluation"}).text.strip()
comment = li_node.find(name='p', attrs={"class": "evalNum"}).text.strip()
import re
price_num = re.findall(r"¥(\d+)", price)[0]
comment_num = re.findall(r"已有(\d+)人评价", comment)[0]
print(title)
print(price, price_num)
print(comment, comment_num)
print("-" * 30)
案例:双色球历史数据
第1页:HTML格式
import requests
from bs4 import BeautifulSoup
res = requests.get(
url="https://m.78500.cn/kaijiang/ssq/",
headers={
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
}
)
soup = BeautifulSoup(res.text, features="html.parser")
parent_area = soup.find(name="article", attrs={'id': "list"})
section_list = parent_area.find_all(name='section', attrs={"class": "item"})
for section in section_list:
title = section.find(name="strong").text
code = section.find(name="p").text
print(title, code)
第2+页:JSON格式
import requests
from bs4 import BeautifulSoup
res = requests.get(
url="https://m.78500.cn/kaijiang/ssq/",
headers={
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
}
)
cookie_dict = res.cookies.get_dict()
print(cookie_dict)
soup = BeautifulSoup(res.text, features="html.parser")
parent_area = soup.find(name="article", attrs={'id': "list"})
section_list = parent_area.find_all(name='section', attrs={"class": "item"})
for section in section_list:
title = section.find(name="strong").text
code = section.find(name="p").text
print(title, code)
第2页(请求与网页不一致)
for i in range(1, 11):
res = requests.get(
url=f"https://m.78500.cn/kaijiang/ssq/?years=list&page={i}",
headers={
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
"X-Requested-With": "XMLHttpRequest",
"Accept": "application/json"
},
cookies=cookie_dict
)
data_dict = res.json()
for row in data_dict['list']:
# print(row['qishu'], "".join(row['result']))
line = f"{row['qishu']}期 {''.join(row['result'])}"
print(line)
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】凌霞软件回馈社区,博客园 & 1Panel & Halo 联合会员上线
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】博客园社区专享云产品让利特惠,阿里云新客6.5折上折
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步