python之tcp粘包以及粘包的解决方法, ftp文件的上传和下载
先来一个案例：
# 粘包案例server.py import socket,time import subprocess phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) ip_port=('127.0.0.1',8080) phone.bind(ip_port) phone.listen(5) conn,addr=phone.accept() #接收参数都大于客户端的字节数，不会粘包 # data1=conn.recv(1024) # data2=conn.recv(1024) data1=conn.recv(5) time.sleep(5) data2=conn.recv(1024) print('第一个包',data1)#b'hello'
print('第二个包',data2)#b'worldSB'
# 粘包案例client.py import socket,time import subprocess phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) ip_port=('127.0.0.1',8080) phone.connect(ip_port) # phone.send('helloworld'.encode('utf-8')) time.sleep(3) phone.send('i am ada'.encode('utf-8'))
先执行服务端，再执行客户端，回过头来看服务端，两段数据明显被打乱了。
# 粘包的概念 # 所谓粘包问题是因为在流传输中，接收方一次接收数据，因为不知道消息之间的界限， # 不知道一次性提取多少字节的数据而造成的不能体现一个完整的消息数据的现象 # 粘包产生的场景： # 双方发送一段数据，有且只有一段数据，就关闭连接，这样就不会出现粘包问题 # 如果发送数据无结构，如文件传输，这样发送方只管发送，接收方只管接收存储就ok，也不用考虑粘包 # 如果双方建立连接，需要在连接后一段时间内发送不同结构数据，就可能粘包 # 粘包产生的原因： # 在tcp流传输中出现，以下从发送和接收两方面来看造成粘包的原因。UDP不会出现粘包，因为它有消息边界 # 1 发送端需要等缓冲区满才发送出去，造成粘包。就是这段数据不够塞满缓存，用下段数据塞满了，丧失了该段数据的完整性 # 2 接收方不及时接收缓冲区的包，造成多个包接收。就是接收了一个包，又接收了几个包，搞到一块了，也丧失了一段数据的完整性 # 粘包的后果： # 丧失了该段数据的完整性，那段数据变多了，或者变少了，变得不完整了 # 如何防止粘包： # 所以我们的任务是保证传输的数据就是那一段==> # 完整性就是数据的开始，结束，数据的长度==> # 所以你想到了统计传输数据字节数，按照字节数传输完整 # 其实，只要传输的个数小于等于接收函数conn.recv(1024)内的参数，都是不会产生粘包的，但是超过了肯定粘包
那如何解决我们的粘包问题呢？用一个模拟命令行操作的案例。
#coding:utf-8 # 这个案例是简单的命令行输入实现，cmd_server.py # 解决思路：利用struct.pack('i',data_size))封装数据长度并字节化作为报头传给客户端，所以先传报头，再传数据 # 在客户端通过struct.unpack('i',baotou)[0]，解压字节变成正常数字来使用，所以先接报头，再接数据 import socket import struct import subprocess phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) #绑定电话卡 ip_port=('127.0.0.1',8081) phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) phone.bind(ip_port) #开机 phone.listen(5) #等待电话 #链接循环 while True: conn,addr=phone.accept() print('client addr',addr) #通讯循环 while True: try: cmd=conn.recv(1024) res=subprocess.Popen(cmd.decode('utf-8'), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out_res=res.stdout.read() err_res=res.stderr.read() data_size=len(out_res)+len(err_res) #发送报头 conn.send(struct.pack('i',data_size)) #发送数据部分 conn.send(out_res) conn.send(err_res) except Exception: break conn.close() phone.close()
# 这个案例是简单的命令行输入实现，cmd_client.py import socket import struct phone = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ip_port = ('127.0.0.1', 8081) phone.connect(ip_port) # 通信循环 while True: # 发消息 cmd = input('>>: ').strip() if not cmd: continue phone.send(bytes(cmd, encoding='utf-8')) #收报头 baotou=phone.recv(4) data_size=struct.unpack('i',baotou)[0] # 收数据 recv_size=0 recv_data=b'' while recv_size < data_size: data=phone.recv(1024) #接收0-1024字节之间任意个数并返回，不是只返回1024个 recv_size+=len(data) recv_data+=data print(recv_data.decode('utf-8')) phone.close()
来基于以上的代码实现一个比较普遍的做法：基于自定义报头
#coding:utf-8 #server.py # 这里的思路是： # 服务端，将长度等未来需要的参数都送进字典结构，并json字符串化，编码为字节传送过去 # 客户端，将接收到的先把字节解码，再反序列化，转换成字典结构，取出变量 #买手机 import socket import struct import json import subprocess phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) #绑定电话卡 ip_port=('192.168.16.114',8081) phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) phone.bind(ip_port) #开机 phone.listen(5) #等待电话 #链接循环 while True: conn,addr=phone.accept() print('client addr',addr) #通讯循环 while True: try: cmd=conn.recv(1024) res=subprocess.Popen(cmd.decode('utf-8'), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out_res=res.stdout.read() err_res=res.stderr.read() data_size=len(out_res)+len(err_res) head_dic={'data_size':data_size} head_json=json.dumps(head_dic) head_bytes=head_json.encode('utf-8') #part1:先发报头的长度 head_len=len(head_bytes) conn.send(struct.pack('i',head_len)) #part2:再发送报头 conn.send(head_bytes) #part3:最后发送数据部分 conn.send(out_res) conn.send(err_res) except Exception: break conn.close() phone.close()
#client.py import socket import struct import json phone = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # 拨通电话 # ip_port = ('127.0.0.1', 8080) ip_port = ('192.168.16.114', 8081) phone.connect(ip_port) # 通信循环 while True: # 发消息 cmd = input('>>: ').strip() if not cmd: continue phone.send(bytes(cmd, encoding='utf-8')) # part1:先收报头的长度 head_struct=phone.recv(4) head_len=struct.unpack('i',head_struct)[0] # part2:再收报头 head_bytes=phone.recv(head_len) head_json=head_bytes.decode('utf-8') head_dic=json.loads(head_json) print(head_dic) data_size = head_dic['data_size'] #part3：收数据 recv_size = 0 recv_data = b'' while recv_size < data_size: data = phone.recv(1024) recv_size += len(data) recv_data += data print(recv_data.decode('utf-8')) phone.close()
这样就比较好的解决了粘包问题。
接下俩，用这种思路完成上传和下载功能。
代码如下：
# upload and download server.py
import socket
import struct
import json
import subprocess
import os
class MYTCPServer:
address_family = socket.AF_INET
socket_type = socket.SOCK_STREAM
allow_reuse_address = True
max_packet_size = 8192
coding='utf-8'
request_queue_size = 5
#上传文件地址
server_dir='/Users/Alex/desktop/file_upload'
def __init__(self, server_address, bind_and_activate=True):
"""Constructor. May be extended, do not override."""
self.server_address=server_address
self.socket = socket.socket(self.address_family,
self.socket_type)
if bind_and_activate:
try:
self.server_bind()
self.server_activate()
except:
self.server_close()
raise
def server_bind(self):
"""Called by constructor to bind the socket.
"""
if self.allow_reuse_address:
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(self.server_address)
self.server_address = self.socket.getsockname()
def server_activate(self):
"""Called by constructor to activate the server.
"""
self.socket.listen(self.request_queue_size)
def server_close(self):
"""Called to clean-up the server.
"""
self.socket.close()
def get_request(self):
"""Get the request and client address from the socket.
"""
return self.socket.accept()
def close_request(self, request):
"""Called to clean up an individual request."""
request.close()
def run(self):
while True:
self.conn,self.client_addr=self.get_request()
print('from client ',self.client_addr)
while True:
try:
head_struct = self.conn.recv(4)
if not head_struct:break
head_len = struct.unpack('i', head_struct)[0]
head_json = self.conn.recv(head_len).decode(self.coding)
head_dic = json.loads(head_json)
print(head_dic)
#head_dic={'cmd':'put','filename':'a.txt','filesize':123123}
cmd=head_dic['cmd']
if hasattr(self,cmd):
func=getattr(self,cmd)
func(head_dic)
except Exception:
break
def put(self,args):
file_path=os.path.normpath(os.path.join(
self.server_dir,
args['filename']
))
filesize=args['filesize']
recv_size=0
print('----->',file_path)
with open(file_path,'wb') as f:
while recv_size < filesize:
recv_data=self.conn.recv(self.max_packet_size)
f.write(recv_data)
recv_size+=len(recv_data)
print('recvsize:%s filesize:%s' %(recv_size,filesize))
tcpserver1=MYTCPServer(('127.0.0.1',9002))
tcpserver1.run()
#下列代码与本题无关
class MYUDPServer:
"""UDP server class."""
address_family = socket.AF_INET
socket_type = socket.SOCK_DGRAM
allow_reuse_address = False
max_packet_size = 8192
coding='utf-8'
def get_request(self):
data, client_addr = self.socket.recvfrom(self.max_packet_size)
return (data, self.socket), client_addr
def server_activate(self):
# No need to call listen() for UDP.
pass
def shutdown_request(self, request):
# No need to shutdown anything.
self.close_request(request)
def close_request(self, request):
# No need to close anything.
pass
# upload and download client.py
import socket import struct import json import os class MYTCPClient: address_family = socket.AF_INET socket_type = socket.SOCK_STREAM allow_reuse_address = False max_packet_size = 8192 coding='utf-8' request_queue_size = 5 def __init__(self, server_address, connect=True): self.server_address=server_address self.socket = socket.socket(self.address_family, self.socket_type) if connect: try: self.client_connect() except: self.client_close() raise def client_connect(self): self.socket.connect(self.server_address) def client_close(self): self.socket.close() def run(self): while True: inp=input(">>: ").strip() if not inp:continue l=inp.split() cmd=l[0] if hasattr(self,cmd): func=getattr(self,cmd) func(l) def put(self,args): cmd=args[0] filename=args[1] if not os.path.isfile(filename): print('file:%s is not exists' %filename) return else: filesize=os.path.getsize(filename) head_dic={'cmd':cmd,'filename':os.path.basename(filename),'filesize':filesize} print(head_dic) head_json=json.dumps(head_dic) head_json_bytes=bytes(head_json,encoding=self.coding) head_struct=struct.pack('i',len(head_json_bytes)) self.socket.send(head_struct) self.socket.send(head_json_bytes) send_size=0 with open(filename,'rb') as f: for line in f: self.socket.send(line) send_size+=len(line) print(send_size) else: print('upload successful') client=MYTCPClient(('127.0.0.1',8080)) client.run()
运行方法：先运行服务端，再运行客户端
客户端所写：
server_dir='/Users/Alex/desktop/file_upload'
所以我在这个路径下建立了file_upload文件夹
然后在/Users/Alex/desktop/hello.mp4 在这里搞了个文件hello.mp4
运行完毕之后，在客户端输入>>:put /Users/Alex/desktop/hello.mp4
就会出现进度数据并出现上传成功
