python: thead and processing

线程：

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
# encoding: utf-8
# 版权所有 2023 涂聚文有限公司
# 许可信息查看：
# 描述：
# Author    : geovindu,Geovin Du 涂聚文.
# IDE       : PyCharm 2023.1 python 311
# Datetime  : 2023/7/16 18:55
# User      : geovindu
# Product   : PyCharm
# Project   : pythonTkinterDemo
# File      : crawlerDouyu.py
# explain   : 学习
 
 
import json
import requests
import os
import threading
 
 
def saveimageUrl(url, dirName):
    """
    线程
    :param url: 链接
    :param dirName: 文件夹名字
    :return:
    """
    # if not os.path.exists('website'):
    # os.mkdir('website')
    try:
        if os.path.exists('website'):
            os.chdir('website')
        if not os.path.exists(f"{dirName}"):
            os.mkdir(dirName)
        content = requests.get(url)
        # print(content.text)
        data = json.loads(content.text)  # 转字典
        # print(data)
 
        cn = 1
        for d in data['data']['rl'][0:30]:  # 切片为前30个数据
            print(d['nn'], d['rs16'], d['rn'])
            #if not d['av']:
                #av=d['Info'][av]
            av = 'https://apic.douyucdn.cn/upload' + '/' + d['av'] + '_big' + '.jpg'
            print(av)
            name = d['nn']
            # 创建多线程
            t = threading.Thread(target=loadAvator, args=(av, dirName, name))
            t.start()
            # img1=Image.open(av)
            # img1=img1.save('img'+cn+'.jpg')
            # imagedata = requests.get(av)
            # with open(fr'{dirName}\{d["nn"]}.jpg','wb') as fp:
            # fp.write(imagedata)
 
    except Exception as ex:
        print(ex)
 
 
def loadAvator(av, dirname, name):
    """
 
    :param av:图片
    :param dirname:文件夹名字
    :param name:博主名称
    :return:
    """
    print("jpg:", fr'{dirname}\{name}.jpg')
    imagedata = requests.get(av)
    try:
        with open(fr'{dirname}\{name}.jpg', 'wb') as fp:
            fp.write(imagedata.content)
            # fp.close()
    except Exception as ex:
        print(ex)

　　

进程：

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# encoding: utf-8
# 版权所有 2023 涂聚文有限公司
# 许可信息查看：
# 描述：
# Author    : geovindu,Geovin Du 涂聚文.
# IDE       : PyCharm 2023.1 python 311
# Datetime  : 2023/7/16 19:10
# User      : geovindu
# Product   : PyCharm
# Project   : pythonTkinterDemo
# File      : crawierProcessing.py
# explain   : 学习
 
 
import multiprocessing
import threading
import time
import json
import requests
import os
import sys
 
 
def saveimageUrl(url, dirName):
    """
    进程  并发
    :param url: 链接
    :param dirName: 文件夹名字
    :return:
    """
    # if not os.path.exists('website'):
    # os.mkdir('website')
    try:
        if os.path.exists('website'):
            os.chdir('website')
        if not os.path.exists(f"{dirName}"):
            os.mkdir(dirName)
        content = requests.get(url)
        # print(content.text)
        data = json.loads(content.text)  # 转字典
        # print(data)
 
        cn = 1
        for d in data['data']['rl'][0:30]:  # 切片为前30个数据
            print(d['nn'], d['rs16'], d['rn'])
            #if not d['av']:
                #av=d['Info'][av]
            av = 'https://apic.douyucdn.cn/upload' + '/' + d['av'] + '_big' + '.jpg'
            print(av)
            name = d['nn']
            # 创建进程
            t = multiprocessing.Process(target=loadAvator, args=(av, dirName, name))
            t.start()
            # img1=Image.open(av)
            # img1=img1.save('img'+cn+'.jpg')
            # imagedata = requests.get(av)
            # with open(fr'{dirName}\{d["nn"]}.jpg','wb') as fp:
            # fp.write(imagedata)
 
    except Exception as ex:
        print(ex)
 
 
def loadAvator(av, dirname, name):
    """
 
    :param av:图片
    :param dirname:文件夹名字
    :param name:博主名称
    :return:
    """
    print("jpg:", fr'{dirname}\{name}.jpg')
    imagedata = requests.get(av)
    try:
        with open(fr'{dirname}\{name}.jpg', 'wb') as fp:
            fp.write(imagedata.content)
            # fp.close()
    except Exception as ex:
        print(ex)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
def dance():
    """
 
    :return:
    """
    while True:
        print('dance')
        time.sleep(0.5)
 
 
def sing():
    """
 
    :return:
    """
    while True:
        print('sing')
        time.sleep(0.5)
 
 
def main():
    """
    线程  是并发  多CPU,多核
    :return:
    """
    '''
    print(multiprocessing.current_process().name)
    print(multiprocessing.current_process().ident)
    p1=multiprocessing.Process(target=dance)
    p2=multiprocessing.Process(target=sing)
    p1.start()
    p2.start()
    print("cpu个数：",multiprocessing.cpu_count)
    '''
 
    for _ in range(10):
        t = threadSing.sing()
        t2 = threadSing.dance()
 
    for _ in range(10):
        p1 = multiprocessing.Process(target=dance)
        p2 = multiprocessing.Process(target=sing)
        p1.start()
        p2.start()

　　

调用：

?
1
2
3
4
5
6
7
8
if not os.path.exists('website'):
    os.mkdir('website')
os.chdir('website')
#Common.crawlerDouyu.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_1/2", 'geovindu')
#Common.crawierProcessing.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_1/2", 'process')
#Common.crawierProcessing.saveimageUrl("https://www.douyu.com/wgapi/ordnc/live/web/room/yzList/1", 'face')
#Common.crawierProcessing.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_194/2", 'sugar')
Common.crawierProcessing.saveimageUrl("https://www.douyu.com/wgapi/ordnc/live/web/room/mixList/2/1008/0/2", 'stars')

　　

posted @   ®Geovin Du Dream Park™  阅读(10)  评论(0)  编辑  收藏  举报
相关博文：
·  python: thread
·  python: 并发编程 (Concurrent Programming)
·  Day 17 17.5 各种线程方法对比下载斗图吧图片
·  aiohttp模块
·  多线程、线程池、多进程、协程
阅读排行：
· 25岁的心里话
· 闲置电脑爆改个人服务器（超详细） #公网映射 #Vmware虚拟网络编辑器
· 基于 Docker 搭建 FRP 内网穿透开源项目（很简单哒）
· 零经验选手，Compose 一天开发一款小游戏！
· 一起来玩mcp_server_sqlite，让AI帮你做增删改查！！
历史上的今天：
2015-07-16 learning sql (second edition) script
2013-07-16 javascript: iframe switchSysBar 左欄打開關閉，兼容各瀏覽器操作
2013-07-16 SQL:exec sp_executesql 用法
< 2025年3月 >
日 一 二 三 四 五 六
23 24 25 26 27 28 1
2 3 4 5 6 7 8
9 10 11 12 13 14 15
16 17 18 19 20 21 22
23 24 25 26 27 28 29
30 31 1 2 3 4 5
点击右上角即可分享
微信分享提示