自动化运维
bin>start.py
conf>config.py
files>
board.out
cpuinfo.out
disk.out
memory.out
nic.out
lib>conf>conf.py
global_settings.py
convert.py
src>plugins
init..py
basic.py
board.py
cpu.py
disk.py
memory.py
nic.py
client.py
script.py
start.py:
from src.script import run
if __name__ == '__main__':
run()
config.py
##### 用户自定制的文件
import os
BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MODE = 'agent' # ssh/salt
# SSH_PORT = 22
# SSH_USERNAME = 'root'
# SSH_PWD = '123'
DEBUG = True
PLUGINS_DICT = {
'basic' : 'src.plugins.basic.Basic',
'cpu' : 'src.plugins.cpu.Cpu',
'disk' : 'src.plugins.disk.Disk',
'board' : 'src.plugins.board.Board',
'memory' : 'src.plugins.memory.Memory',
}
API_URL = 'http://127.0.0.1:8000/asset/'
conf.py
from lib.conf import global_settings
from conf import config
class Settings():
def __init__(self):
#### 集成全局的配置
for k in dir(global_settings):
if k.isupper():
v = getattr(global_settings, k)
setattr(self, k, v)
#### 集成自定制的配置
for k in dir(config):
if k.isupper():
v = getattr(config, k)
setattr(self, k, v)
setting = Settings()
-----------------------------------
global_setting.py
#### ##### 全局默认配置文件
EMAIL_PORT = 25
USER = 'admin'
-----------------------------------
convert.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
def convert_to_int(value,default=0):
try:
result = int(value)
except Exception as e:
result = default
return result
def convert_mb_to_gb(value,default=0):
try:
value = value.strip('MB')
result = int(value)
except Exception as e:
result = default
return result
__init__.py
from lib.conf.conf import setting
import importlib
import traceback
#### 管理采集插件的类
class PluginsManager():
def __init__(self, hostname=None):
self.hostname = hostname
self.plugins_dict = setting.PLUGINS_DICT
self.mode = setting.MODE
self.debug = setting.DEBUG
if self.mode == 'ssh':
self.ssh_port = setting.SSH_PORT
self.ssh_username = setting.SSH_USERNAME
self.ssh_pwd = setting.SSH_PWD
#### c从配置文件中读取要采集的插件信息， 并且执行
def execute(self):
'''
PLUGINS_DICT = {
'basic' : 'src.plugins.basic.Basic',
'cpu' : 'src.plugins.cpu.Cpu',
'disk' : 'src.plugins.disk.Disk',
}
'''
response = {}
for k, v in self.plugins_dict.items():
ret = {"status":None, 'data':None}
'''
k: basic
v: src.plugins.basic.Basic
'''
try:
module_path, class_name = v.rsplit('.',1) ### ['src.plugins.basic', 'Basic']
### 如何将字符串形式的模块导入进来
m = importlib.import_module(module_path)
cls = getattr(m, class_name)
res = cls().process(self.command, self.debug)
ret['status'] = 10000
ret['data'] = res
except Exception as e:
ret['status'] = 100001
ret['data'] = traceback.format_exc()
response[k] = ret
return response
def command(self, cmd):
if self.mode == 'agent':
import subprocess
res = subprocess.getoutput(cmd)
return res
elif self.mode == 'ssh':
import paramiko
ssh = paramiko.SSHClient()
# 允许连接不在know_hosts文件中的主机
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# 连接服务器
ssh.connect(hostname=self.hostname, port=self.ssh_port, username=self.ssh_username, password=self.ssh_pwd)
# 执行命令
stdin, stdout, stderr = ssh.exec_command(cmd)
# 获取命令结果
result = stdout.read()
# 关闭连接
ssh.close()
return result
elif self.mode == 'salt':
import salt.client
local = salt.client.LocalClient()
result = local.cmd(self.hostname, 'cmd.run', [cmd])
return result
else:
raise Exception ('只支持agent/ssh/salt模式')
basic.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
class Basic(object):
def __init__(self):
pass
def process(self, command_func, debug):
if debug:
output = {
'os_platform': "linux",
'os_version': "CentOS release 6.6 (Final)\nKernel \r on an \m",
'hostname': 'c2000.com'
}
else:
output = {
'os_platform': command_func("uname").strip(),
'os_version': command_func("cat /etc/issue").strip().split('\n')[0],
'hostname': command_func("hostname").strip(),
}
return output
board.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import os
from lib.conf.conf import setting
class Board(object):
def __init__(self):
pass
def process(self, command_func, debug):
if debug:
output = open(os.path.join(setting.BASEDIR, 'files/board.out'), 'r', encoding='utf-8').read()
else:
output = command_func("sudo dmidecode -t1")
return self.parse(output)
def parse(self, content):
result = {}
key_map = {
'Manufacturer': 'manufacturer',
'Product Name': 'model',
'Serial Number': 'sn',
}
for item in content.split('\n'):
row_data = item.strip().split(':')
if len(row_data) == 2:
if row_data[0] in key_map:
result[key_map[row_data[0]]] = row_data[1].strip() if row_data[1] else row_data[1]
return result
cpu.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import os
from lib.conf.conf import setting
class Cpu(object):
def __init__(self):
pass
def process(self, command_func, debug):
if debug:
output = open(os.path.join(setting.BASEDIR, 'files/cpuinfo.out'), 'r', encoding='utf-8').read()
else:
output = command_func("cat /proc/cpuinfo")
return self.parse(output)
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
response = {'cpu_count': 0, 'cpu_physical_count': 0, 'cpu_model': ''}
cpu_physical_set = set()
content = content.strip()
for item in content.split('\n\n'):
for row_line in item.split('\n'):
key, value = row_line.split(':')
key = key.strip()
if key == 'processor':
response['cpu_count'] += 1
elif key == 'physical id':
cpu_physical_set.add(value)
elif key == 'model name':
if not response['cpu_model']:
response['cpu_model'] = value
response['cpu_physical_count'] = len(cpu_physical_set)
return response
disk.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import re
import os
from lib.conf.conf import setting
class Disk(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(setting.BASEDIR, 'files/disk.out'), 'r', encoding='utf-8').read()
else:
output = command_func("sudo MegaCli -PDList -aALL")
return self.parse(output)
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
response = {}
result = []
for row_line in content.split("\n\n\n\n"):
result.append(row_line)
for item in result:
temp_dict = {}
for row in item.split('\n'):
if not row.strip():
continue
if len(row.split(':')) != 2:
continue
key, value = row.split(':')
name = self.mega_patter_match(key)
if name:
if key == 'Raw Size':
raw_size = re.search('(\d+\.\d+)', value.strip())
if raw_size:
temp_dict[name] = raw_size.group()
else:
raw_size = '0'
else:
temp_dict[name] = value.strip()
if temp_dict:
response[temp_dict['slot']] = temp_dict
return response
@staticmethod
def mega_patter_match(needle):
grep_pattern = {'Slot': 'slot', 'Raw Size': 'capacity', 'Inquiry': 'model', 'PD Type': 'pd_type'}
for key, value in grep_pattern.items():
if needle.startswith(key):
return value
return False
memory.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import os
from lib import convert
from lib.conf.conf import setting
class Memory(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(setting.BASEDIR, 'files/memory.out'), 'r', encoding='utf-8').read()
else:
output = command_func("sudo dmidecode -q -t 17 2>/dev/null")
return self.parse(output)
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
ram_dict = {}
key_map = {
'Size': 'capacity',
'Locator': 'slot',
'Type': 'model',
'Speed': 'speed',
'Manufacturer': 'manufacturer',
'Serial Number': 'sn',
}
devices = content.split('Memory Device')
for item in devices:
item = item.strip()
if not item:
continue
if item.startswith('#'):
continue
segment = {}
lines = item.split('\n\t')
for line in lines:
if not line.strip():
continue
if len(line.split(':')):
key, value = line.split(':')
else:
key = line.split(':')[0]
value = ""
if key in key_map:
if key == 'Size':
segment[key_map['Size']] = convert.convert_mb_to_gb(value, 0)
else:
segment[key_map[key.strip()]] = value.strip()
ram_dict[segment['slot']] = segment
return ram_dict
nic.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import os
import re
from lib.conf.conf import setting
class Nic(object):
def __init__(self):
pass
@classmethod
def initial(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(setting.BASEDIR, 'files/nic.out'), 'r', encoding='utf-8').read()
interfaces_info = self._interfaces_ip(output)
else:
interfaces_info = self.linux_interfaces(command_func)
self.standard(interfaces_info)
return interfaces_info
def linux_interfaces(self, command_func):
'''
Obtain interface information for *NIX/BSD variants
'''
ifaces = dict()
ip_path = 'ip'
if ip_path:
cmd1 = command_func('sudo {0} link show'.format(ip_path))
cmd2 = command_func('sudo {0} addr show'.format(ip_path))
ifaces = self._interfaces_ip(cmd1 + '\n' + cmd2)
return ifaces
def which(self, exe):
def _is_executable_file_or_link(exe):
# check for os.X_OK doesn't suffice because directory may executable
return (os.access(exe, os.X_OK) and
(os.path.isfile(exe) or os.path.islink(exe)))
if exe:
if _is_executable_file_or_link(exe):
# executable in cwd or fullpath
return exe
# default path based on busybox's default
default_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin'
search_path = os.environ.get('PATH', default_path)
path_ext = os.environ.get('PATHEXT', '.EXE')
ext_list = path_ext.split(';')
search_path = search_path.split(os.pathsep)
if True:
# Add any dirs in the default_path which are not in search_path. If
# there was no PATH variable found in os.environ, then this will be
# a no-op. This ensures that all dirs in the default_path are
# searched, which lets salt.utils.which() work well when invoked by
# salt-call running from cron (which, depending on platform, may
# have a severely limited PATH).
search_path.extend(
[
x for x in default_path.split(os.pathsep)
if x not in search_path
]
)
for path in search_path:
full_path = os.path.join(path, exe)
if _is_executable_file_or_link(full_path):
return full_path
return None
def _number_of_set_bits_to_ipv4_netmask(self, set_bits): # pylint: disable=C0103
'''
Returns an IPv4 netmask from the integer representation of that mask.
Ex. 0xffffff00 -> '255.255.255.0'
'''
return self.cidr_to_ipv4_netmask(self._number_of_set_bits(set_bits))
def cidr_to_ipv4_netmask(self, cidr_bits):
'''
Returns an IPv4 netmask
'''
try:
cidr_bits = int(cidr_bits)
if not 1 <= cidr_bits <= 32:
return ''
except ValueError:
return ''
netmask = ''
for idx in range(4):
if idx:
netmask += '.'
if cidr_bits >= 8:
netmask += '255'
cidr_bits -= 8
else:
netmask += '{0:d}'.format(256 - (2 ** (8 - cidr_bits)))
cidr_bits = 0
return netmask
def _number_of_set_bits(self, x):
'''
Returns the number of bits that are set in a 32bit int
'''
# Taken from http://stackoverflow.com/a/4912729. Many thanks!
x -= (x >> 1) & 0x55555555
x = ((x >> 2) & 0x33333333) + (x & 0x33333333)
x = ((x >> 4) + x) & 0x0f0f0f0f
x += x >> 8
x += x >> 16
return x & 0x0000003f
def _interfaces_ip(self, out):
'''
Uses ip to return a dictionary of interfaces with various information about
each (up/down state, ip address, netmask, and hwaddr)
'''
ret = dict()
right_keys = ['name', 'hwaddr', 'up', 'netmask', 'ipaddrs']
def parse_network(value, cols):
'''
Return a tuple of ip, netmask, broadcast
based on the current set of cols
'''
brd = None
if '/' in value: # we have a CIDR in this address
ip, cidr = value.split('/') # pylint: disable=C0103
else:
ip = value # pylint: disable=C0103
cidr = 32
if type_ == 'inet':
mask = self.cidr_to_ipv4_netmask(int(cidr))
if 'brd' in cols:
brd = cols[cols.index('brd') + 1]
return (ip, mask, brd)
groups = re.compile('\r?\n\\d').split(out)
for group in groups:
iface = None
data = dict()
for line in group.splitlines():
if ' ' not in line:
continue
match = re.match(r'^\d*:\s+([\w.\-]+)(?:@)?([\w.\-]+)?:\s+<(.+)>', line)
if match:
iface, parent, attrs = match.groups()
if 'UP' in attrs.split(','):
data['up'] = True
else:
data['up'] = False
if parent and parent in right_keys:
data[parent] = parent
continue
cols = line.split()
if len(cols) >= 2:
type_, value = tuple(cols[0:2])
iflabel = cols[-1:][0]
if type_ in ('inet',):
if 'secondary' not in cols:
ipaddr, netmask, broadcast = parse_network(value, cols)
if type_ == 'inet':
if 'inet' not in data:
data['inet'] = list()
addr_obj = dict()
addr_obj['address'] = ipaddr
addr_obj['netmask'] = netmask
addr_obj['broadcast'] = broadcast
data['inet'].append(addr_obj)
else:
if 'secondary' not in data:
data['secondary'] = list()
ip_, mask, brd = parse_network(value, cols)
data['secondary'].append({
'type': type_,
'address': ip_,
'netmask': mask,
'broadcast': brd,
})
del ip_, mask, brd
elif type_.startswith('link'):
data['hwaddr'] = value
if iface:
if iface.startswith('pan') or iface.startswith('lo') or iface.startswith('v'):
del iface, data
else:
ret[iface] = data
del iface, data
return ret
def standard(self, interfaces_info):
for key, value in interfaces_info.items():
ipaddrs = set()
netmask = set()
if not 'inet' in value:
value['ipaddrs'] = ''
value['netmask'] = ''
else:
for item in value['inet']:
ipaddrs.add(item['address'])
netmask.add(item['netmask'])
value['ipaddrs'] = '/'.join(ipaddrs)
value['netmask'] = '/'.join(netmask)
del value['inet']
client.py
from lib.conf.conf import setting
import json
import os
from src.plugins import PluginsManager
import requests
class Base():
def postData(self, data):
requests.post(setting.API_URL, data=json.dumps(data))
class Agent(Base):
def collect(self):
res = PluginsManager().execute()
hostname = res['basic']['data']['hostname'] ### c2000.com
ret = open(os.path.join(setting.BASEDIR, 'conf/cert'), 'r', encoding='utf8').read()
if not ret:
with open(os.path.join(setting.BASEDIR, 'conf/cert'), 'w', encoding='utf8') as fp:
fp.write(hostname)
else:
res['basic']['data']['hostname'] = ret
for k, v in res.items():
print(k, v)
self.postData(res)
class SSHSalt(Base):
def getHostnames(self):
hostnames = requests.get(setting.API_URL)
# return hostnames
return ['c1.com', 'c2.com']
def run(self, hostname):
res = PluginsManager(hostname=hostname).execute()
self.postData(res)
def collect(self):
from concurrent.futures import ThreadPoolExecutor
p = ThreadPoolExecutor(10)
hostnames = self.getHostnames()
for hostname in hostnames:
p.submit(self.run, hostname)
script.py
from lib.conf.conf import setting
from src.client import Agent, SSHSalt
def run():
mode = setting.MODE
if mode == 'agent':
obj = Agent()
else:
obj = SSHSalt()
obj.collect()
程序的道路上一去不复返
