python 使用 paramiko 和 threading 模块 在本地windows上实现同时与多个远程Linux机子连接操作cmd命令和上传(下载)单个文件或者整个目录

关键知识:　 

　　paramiko 模块, stat模块,os模块

　　threading 模块 --> 实现同时与多个远程上传和下载单个文件或者整个目录

　　paramiko 模块 --> 实现链接远程,传输数据

　　stat中的  stat.S_ISDIR(remote_path_mode)  -->  判断远程是目录还是文件

　　os中的 os.walk(path)   --> 提取本地目录中的所有文件和子目录

 　　

定义一个基础类Connection,继承 threading.Thread

import paramiko, datetime, os, threading, re, stat


class Connection(threading.Thread):

    def __init__(self, hostname=None, port=None, password=None, username=None):
        super(Connection, self).__init__()
        self.hostname = hostname
        self.port = port
        self.password = password
        self.username = username
        self.str_host = '%s:%s' % (self.hostname, self.port)

    def run(self):
        pass

 

定义一个cmd执行类CMDThread,继承Connection. CMDThread类实例化后会处理cmd相关命令

class CMDThread(Connection):

    def __init__(self, hostname=None, port=None, password=None, username=None, echo_cmd=None):
        super(CMDThread, self).__init__(hostname, port, password, username,)
        self.echo_cmd = echo_cmd

    def run(self):
        paramiko.util.log_to_file('MyFabric.log')
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        try:
            ssh.connect(hostname=self.hostname, port=self.port, username=self.username, password=self.password)
            stdin, stdout, stderr = ssh.exec_command(self.echo_cmd)
            res, err = stdout.read(), stderr.read()
            result = res if res else err
            print(self.str_host.center(60, '-'))
            print(result.decode())
            ssh.close()
        except Exception as e:
            print(e)

 

定义一个上传文件的类PutThread,继承Connection,PutThread实例化, 先判断本地要上传的是单个文件还是整个目录,然后链接远程上传至远程指定位置.

class PutThread(Connection):

    def __init__(self, hostname=None, password=None, username=None, port=None, local_path=None, remote_path=None):
        super(PutThread, self).__init__(hostname, password, username, port)
        self.local_path = local_path
        self.remote_path = remote_path

    def run(self):
        paramiko.util.log_to_file('MyFabric.log')
        t = paramiko.Transport((self.hostname, self.port))
        t.connect(username=self.username, password=self.password)
        sftp = paramiko.SFTPClient.from_transport(t)
        if os.path.isfile(self.local_path):
            filename = os.path.split(self.local_path)[-1]
            remote_file = os.path.join(self.remote_path, filename).replace('\\', '/')
            try:
                sftp.put(self.local_path, remote_file)
                print('[*] Put dir %s success %s --%s' % (filename, datetime.datetime.now(), self.str_host))
                t.close()
            except Exception as e:
                print('[*] Put file %s failed %s Err:%s --%s' % (filename, datetime.datetime.now(), e, self.str_host))
        elif os.path.isdir(self.local_path):
            local_dir_root = os.path.split(self.local_path)[0].replace('\\', '/')
            dir_name = os.path.split(self.local_path)[-1]
            remote_dir_root = self.remote_path
            remote_files_path = os.path.join(remote_dir_root, dir_name).replace('\\', '/')
            try:
                try:
                    sftp.stat(remote_files_path)
                except IOError:
                    print('mkdir remote dir [%s]' % remote_files_path)
                    sftp.mkdir(remote_files_path)
                for root, dirs, files in os.walk(self.local_path):
                    if local_dir_root:          # 这里判断输入的self.local_path是不是abspath, 然后做相应的路径字符串拼接
                        remote_root = re.sub(local_dir_root, remote_dir_root, root.replace('\\', '/'))
                    else:
                        remote_root = os.path.join(remote_dir_root, root).replace('\\', '/')
                    if dirs:
                        for name in dirs:
                            mk_remote_path = os.path.join(remote_root, name).replace('\\', '/') 
                            sftp.mkdir(mk_remote_path)
                    if files:
                        for filename in files:
                            local_file = os.path.join(root, filename)
                            remote_file = os.path.join(remote_root, filename).replace('\\', '/')
                            sftp.put(local_file, remote_file)
                print('[*] Put dir %s success %s --%s' % (dir_name, datetime.datetime.now(), self.str_host))
                t.close()
            except Exception as e:
                print('[*] Put dir %s failed  %s Err:%s --%s' % (dir_name, datetime.datetime.now(), e, self.str_host))

 

定义一个下载文件的类GetThread, 继承Connection.GetThread实例化后,会先链接远程,判断远程下载的是单个文件还是整个目录, 然后在本地指定位置建立一个以远程hostname命名的文件夹,来存放从该远程下载的文件或整个目录

class GetThread(Connection):
    """
    'local_path' must be abspath
    """
    def __init__(self, hostname=None, password=None, username=None, port=None, local_path=None, remote_path=None):
        super(GetThread, self).__init__(hostname, password, username, port)
        self.local_path = local_path
        self.remote_path = remote_path

    def run(self):
        paramiko.util.log_to_file('MyFabric.log')
        t = paramiko.Transport((self.hostname, self.port))
        t.connect(username=self.username, password=self.password)
        sftp = paramiko.SFTPClient.from_transport(t)
        remote_path_mode = sftp.stat(self.remote_path).st_mode     # 获取远程文件的类型st_mode
        if stat.S_ISDIR(remote_path_mode):                         # 使用stat模块的S_ISDIR方法判断远程文件的st_mode是否是目录类型
            all_files = get_all_files(sftp, self.remote_path)      # get_all_files函数用来从远程目录提取该目录下所有文件的路径
            try:
                for remote_file in all_files:
                    local_root_path = self.local_path + '/' + self.hostname  # 这里做一个本地路径的字符串拼接,把远程hostname拼进去,为后面建立相应的文件夹做准备
                    remote_root_path = os.path.split(self.remote_path)[0]
                    local_file = re.sub(remote_root_path, local_root_path, remote_file)  # 拼接好文件的完整路径
                    root = os.path.split(local_file)[0]                                  # 这个是该文件的根目录
                    check_dirs(root)                    # check_dirs函数用来检查该文件的根目录root是否在本地存在,如果不存在就建立该目录
                    sftp.get(remote_file, local_file)
                print('[*] Get dir %s success %s --%s' % (self.remote_path,
                                                          datetime.datetime.now(),
                                                          self.str_host))
            except Exception as e:
                print('[*] Get dir %s failed %s Err:%s --%s' % (self.remote_path,
                                                                datetime.datetime.now(),
                                                                e,
                                                                self.str_host))
        else:
            filename = os.path.split(self.remote_path)[-1]
            local_root_path = self.local_path + '/' + self.hostname
            local_file = local_root_path + '/' + filename
            check_dirs(local_root_path)    # 同上检查目录是否存在,不存在就建立目录
            try:
                sftp.get(self.remote_path, local_file)
                print('[*] Get file %s success %s --%s' % (filename, datetime.datetime.now(), self.str_host))
            except Exception as e:
                print('[*] Get file %s failed %s Err:%s --%s' % (filename, datetime.datetime.now(), e, self.str_host))
        t.close()

 

其中get_all_files用来从远程目录提取该目录下所有文件的路径.

def get_all_files(sftp, remote_dir):
    """
    get all abspath of files in remote dir
    :param sftp:
    :param remote_dir:
    :return:
    """
    all_files = []
    if remote_dir[-1] == '/':
        remote_dir = remote_dir[0:-1]
    files = sftp.listdir_attr(remote_dir)          # 获取目录下的所有文件和子目录stat信息
    for i in files:
        file_path = remote_dir + '/' + i.filename  # 拼接完整的文件路径
        if stat.S_ISDIR(i.st_mode):                # 用stat模块的S_IDDIR方法判断文件是否是目录 
            all_files.extend(get_all_files(sftp, file_path)) # 如果是子目录就递归处理此目录
        else:
            all_files.append(file_path)            # 如果不是子目录,就把文件路径加到all_files这个列表中
    return all_files

注意: 其中列表操作extend和append的区别,extend()的参数只能是列表,append()的参数什么都可以,具体可自行百度.

 

其中check_dirs 函数是在本地windows上检查目录是否存在,不存在就创建目录

def check_dirs(path):
    """
    Check the root of 'path'  and create the root without existing
    :param path:
    :return:
    """
    dirs = get_dirs_path(path)[::-1]   # get_dirs_path函数用来提取一个目录的所有层级根目录路径包括自己本身的路径
    for i in dirs:
        if not os.path.exists(i):
            os.mkdir(i)

get_dirs_path(path)获取到的是类似这样的列表['c:/Desktop/test' , 'c:/Desktop', 'c:'],  [::-1]这个是把列表顺序反转, --->['c:', 'c:/Desktop', 'c:/Desktop/test']

其中 get_dirs_path函数用来提取一个目录的所有层级根目录路径包括自己本身的路径, 类似上面的列表

def get_dirs_path(path):    # 这里的path必须是abspath
    """
    gets the root directory at all levels of path(including 'path')
    example: 'c:/Desktop/test/' --> ['c:/Desktop/test' , 'c:/Desktop', 'c:']
    'path' must be abspath on windows
    """
    dirs = []
    if path[-1] == '/':
        path = path[0:-1]
    dirs.append(path)
    lis = os.path.split(path)   # 把根目录路径和目录名字分开,然后一层一层扒开处理直到最后出现类似这样的列表['c:', '']这个就是最终扒开的不能再扒了,再扒就进入死循环了,所以下面要在出现这个情况的时候终止递归
    path = lis[0]
    if lis[-1]:
        dirs.extend(get_dirs_path(path))    # 这里也是用递归方法处理一个路径
    return dirs

 

下面就是主逻辑了

def cmd_handler():
    hosts = get_hosts()
    echo_cmd = input('Enter echo cmd:').strip()
    obj = []
    for host in hosts:
        t = CMDThread(host['hostname'], host['port'], host['password'], host['username'], echo_cmd)
        t.start()
        obj.append(t)
    for i in obj:
        i.join()
    print('end'.center(60, '-'))


def put_handler():
    transport(tag='put')


def get_handler():
    transport(tag='get')


def transport(tag=None):
    hosts = get_hosts()
    while True:
        remote_path = input('remote path >>').strip().replace('\\', '/')
        local_path = input('local path >>').strip().replace('\\', '/')
        if local_path and remote_path:
            break
        if not local_path or not remote_path:
            continue
    obj = []
    for host in hosts:
        if tag == 'get':
            t = GetThread(host['hostname'], host['port'], host['password'], host['username'], local_path, remote_path)
        elif tag == 'put':
            t = PutThread(host['hostname'], host['port'], host['password'], host['username'], local_path, remote_path)
        t.start()
        obj.append(t)
    for i in obj:
        i.join()
    print('end'.center(60, '-'))


def get_hosts():
    hosts = []
    with open('hosts_setting', 'r') as f:
        dic = {}
        for line in f:
            date = line.strip().split()
            dic['hostname'] = date[0].split(':')[0]
            dic['port'] = int(date[0].split(':')[-1])
            dic['username'] = date[1]
            dic['password'] = date[-1]
            hosts.append(dic)
            dic = {}
    return hosts

if __name__ == '__main__':
    while True:
        print('''
        menu:
            1.cmd
            2.put
            3.get
            4.quit
        ''')
        menu = {
            '1': cmd_handler,
            '2': put_handler,
            '3': get_handler,
            '4': exit
        }
        choice = input('Choice handler:').strip()
        if choice in menu:
            menu[choice]()

 

有个小问题就是, 如果遇到目录里面有空的子目录,  上传可以把空子目录结构一起复制到远程,  但下载不会把空的子目录结构也复制到本地, 有需要的亲,自己可以动手写,这里就不多说了.