python学习之路

 1 from multiprocessing import Pool
 2 from gevent import monkey; monkey.patch_all()
 3 import gevent
 4 # -*- coding=utf-8 -*-
 5 from urllib import request
 6 from multiprocessing import Process
 7 import gevent,time
 8 from gevent import monkey
 9 monkey.patch_all()
10 import sys
11 def fetch(url):
12     try:
13         # s = request.Session()
14         # r = s.get(url, timeout=1)  # 在这里抓取页面
15         resp = request.urlopen(url)
16         data = resp.read()
17     except Exception as e:
18         print(e)
19     return ''
20 def process_start(url_list):
21     tasks = []
22     for url in url_list:
23         tasks.append(gevent.spawn(fetch, url))
24     gevent.joinall(tasks)  # 使用协程来执行
25 def task_start(filepath, flag=8):  # 每10W条url启动一个进程
26     with open(filepath, 'r') as reader:  # 从给定的文件中读取url
27         url = reader.readline().strip()
28         url_list = []  # 这个list用于存放协程任务
29         i = 0  # 计数器，记录添加了多少个url到协程队列
30         while url != '':
31             i += 1
32             url_list.append(url)  # 每次读取出url，将url添加到队列
33             if i == flag:  # 一定数量的url就启动一个进程并执行
34                 p = Process(target=process_start, args=(url_list,))
35                 p.start()
36                 url_list = []  # 重置url队列
37                 i = 0  # 重置计数器
38             url = reader.readline().strip()
39         if url_list:  # 若退出循环后任务队列里还有url剩余
40             p = Process(target=process_start, args=(url_list,))  # 把剩余的url全都放到最后这个进程来执行
41             p.start()
42 if __name__ == '__main__':
43     print("time",time.time())
44     task_start('./testData.txt')  # 读取指定文件

View Code

io多路复用： (协程)当程序遇到io操作，如从网络接收数据或者从磁盘读数据，这个过程需要磁盘或者网络很长时间的返回，所以程序就切换，就不占用cpu。进程申请打开一个文件，操作系统返回一个文件描述符，操作系统通过文件描述符操作文件对象（文件句柄）。

如网络：socket.recv等于调用操作系统去监听并等待数据，数据返回后，先到内核空间，然后复制到用户空间。

正式因为这两个阶段，linux系统产生了下面五种网络模式的方案。
- 阻塞 I/O（blocking IO）
- 非阻塞 I/O（nonblocking IO）
- I/O 多路复用（ IO multiplexing）
- 信号驱动 I/O（ signal driven IO）
- 异步 I/O（asynchronous IO）

注：由于signal driven IO在实际中并不常用，所以我这只提及剩下的四种IO Model。

第十天---twisted---redis---rabbitmq

windows下redis安装 https://www.cnblogs.com/jaign/articles/7920588.html

windows下rabbitmq安装 http://blog.csdn.net/hzw19920329/article/details/53156015from sqlalchemy import create_engine

http://www.cnblogs.com/alex3714/articles/5248247.html

第十一天 sqlalchemy

一对多

 1 from sqlalchemy.ext.declarative import declarative_base
 2 from sqlalchemy import Column, Integer, String, ForeignKey,func,Table
 3 from  sqlalchemy.orm import sessionmaker, relationship
 4 Base = declarative_base() #生成一个SqlORM 基类
 5 mima = '!Saiyun12#$'
 6 Host2Group = Table('host_2_group',Base.metadata,
 7                    Column('host_id',ForeignKey('hosts.id'),primary_key=True),
 8                    Column('goup_id',ForeignKey('group.id'),primary_key=True),
 9                    )
10 engine = create_engine("mysql+pymysql://root:%s@localhost/testdb" % mima,encoding='utf-8', echo=True)
11 class Host(Base):
12     __tablename__ = 'hosts'
13     id = Column(Integer,primary_key=True,autoincrement=True)
14     hostname = Column(String(64),unique=True,nullable=False)
15     ip_addr = Column(String(128),unique=True,nullable=False)
16     port = Column(Integer,default=22)
17     group_id = Column(Integer,ForeignKey('group.id'))
18     group = relationship("Group",backref='host_list')
19 class Group(Base):
20     __tablename__ = 'group'
21     id = Column(Integer,primary_key=True)
22     name = Column(String(64),unique=True,nullable=False)
23 Base.metadata.create_all(engine) #创建所有表结构
24 if __name__ == '__main__':
25     SessionCls = sessionmaker(bind=engine) #创建与数据库的会话session class ,注意,这里返回给session的是个class,不是实例
26     session = SessionCls()
27     objs = session.query(Host,func.count(Group.name)).join(Host.group).group_by(Group.name).all()  #join操作
28     print("---->",objs)
29     session.commit()  # 提交
30 ''' obj对应的原声sql
31 SELECT hosts.id AS hosts_id, hosts.hostname AS hosts_hostname, hosts.ip_addr AS hosts_ip_addr, hosts.port AS hosts_port, 
32 hosts.group_id AS hosts_group_id, count(`group`.name) AS count_1 
33 FROM hosts INNER JOIN `group` ON `group`.id = hosts.group_id GROUP BY `group`.name
34  ####select * from hosts right join testdb.group on hosts.group_id = testdb.group.id
35 '''

View Code

多对多

 1 from sqlalchemy import create_engine
 2 from sqlalchemy.ext.declarative import declarative_base
 3 from sqlalchemy import Column, Integer, String, ForeignKey,func,Table
 4 from  sqlalchemy.orm import sessionmaker, relationship
 5 Base = declarative_base() #生成一个SqlORM 基类
 6 mima = '!Saiyun12#$'
 7 Host2Group = Table('host_2_group',Base.metadata,
 8                    Column('host_id',ForeignKey('hosts.id'),primary_key=True),
 9                    Column('goup_id',ForeignKey('group.id'),primary_key=True),
10                    )
11 engine = create_engine("mysql+pymysql://root:%s@localhost/testdb" % mima,encoding='utf-8', echo=True)
12 class Host(Base):
13     __tablename__ = 'hosts'
14     id = Column(Integer,primary_key=True,autoincrement=True)
15     hostname = Column(String(64),unique=True,nullable=False)
16     ip_addr = Column(String(128),unique=True,nullable=False)
17     port = Column(Integer,default=22)
18     groups = relationship('Group',secondary=Host2Group,backref='host_list')
19     # group_id = Column(Integer,ForeignKey('group.id'))
20     # group = relationship("Group",backref='host_list')
21 class Group(Base):
22     __tablename__ = 'group'
23     id = Column(Integer,primary_key=True)
24     name = Column(String(64),unique=True,nullable=False)
25 Base.metadata.create_all(engine) #创建所有表结构
26 if __name__ == '__main__':
27     SessionCls = sessionmaker(bind=engine) #创建与数据库的会话session class ,注意,这里返回给session的是个class,不是实例
28     session = SessionCls()
29     # g1= Group(name='g1')
30     # g2 = Group(name='g2')
31     # g3 = Group(name='g3')
32     # g4 = Group(name='g4')
33     # session.add_all([g1,g2,g3,g4])
34     # h1 = Host(hostname='h1',ip_addr='192.168.1.56',port=1000)
35     # h2 = Host(hostname='h2', ip_addr='192.168.1.57', port=1002)
36     # h3 = Host(hostname='h3', ip_addr='192.168.1.58', port=1003)
37     # session.add_all([h1,h2,h3])
38     groups=session.query(Group).all()
39     h1 = session.query(Host).filter(Host.hostname=='h1').first()
40     h1.groups = groups  #为已经存在的host和group表创建多对多关系
41 
42     # objs = session.query(Host,func.count(Group.name)).join(Host.group).group_by(Group.name).all()  #join操作
43     # print("---->",objs)
44     session.commit()  # 提交

View Code

nihaohelloworld

xiao

feng

posted @ 2018-02-12 17:28 jianxiaofeng 阅读(222) 评论(0) 编辑收藏举报

刷新页面返回顶部

python学习之路

面试 https://github.com/taizilongxu/interview_python

老男孩教育Python自动化2.0课程课件目录 http://www.cnblogs.com/alex3714/articles/5885096.html alex

xiao

feng

公告