巡风xunfeng代码研究---threading线程(nascan)
一、
def masscan(self, ip): try: if len(ip) == 0: return sys.path.append(sys.path[0] + "/plugin") m_scan = __import__("masscan") result = m_scan.run(ip, self.masscan_path, self.masscan_rate) return result except Exception, e: print e print 'No masscan plugin detected'
我们知道import语句是用来导入外部模块的，当然还有from...import...也可以，但是其实import实际上是使用builtin函数__import__来工作的。
在一些程序中，我们可以动态地去调用函数，如果我们知道模块的名称(字符串)的时候，我们可以很方便的使用动态调用。 然后本例子里直接导入对应的module后运行，
二、
def scan_start(self): for i in range(self.thread): # 开始扫描 t = ThreadNum(self.queue) t.setDaemon(True) t.mode = self.mode t.config_ini = self.config_ini t.statistics = self.statistics t.start() self.queue.join()
三、开启多线程的类
AC_PORT_LIST = {} MASSCAN_AC = 0 class ThreadNum(threading.Thread): def __init__(self, queue): threading.Thread.__init__(self) self.queue = queue def run(self): while True: if self.queue.empty(): break try: task_host = self.queue.get() if self.mode: port_list = AC_PORT_LIST[task_host] else: port_list = self.config_ini['Port_list'].split('|')[1].split('\n') _s = scan.scan(task_host, port_list) _s.config_ini = self.config_ini # 提供配置信息 _s.statistics = self.statistics # 提供统计信息 _s.run() except Exception,e: print e finally: self.queue.task_done()
每一个scan.scan扫描方法执行一次，生成一个线程
四、 Scan方法讲解(nascan,非mascan)
1) 扫描端口:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ---#定义socket类型，网络通信，TCP
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) --默认的socket选项不够用的时候，就必须要使用setsockopt来调整。就是使用setsockopt。
---有三个参数： level：选项定义的层次。支持SOL_SOCKET、IPPROTO_TCP、IPPROTO_IP和IPPROTO_IPV6。
optname：需设置的选项。
value：设置选项的值。
sock.connect((self.ip, self.port)) --#要连接的IP与端口
time.sleep(0.2)
self.banner = sock.recv(1024) --#把接收的数据定义为变量
sock.close() --然后关闭连接
if len(self.banner) <= 2: --前两个字节是换行符 \r \n
self.banner = 'NULL'
banner = ''
hostname = self.ip2hostname(self.ip) -- 将IP转换为主机名 (此功能是自动发现主机)
time_ = datetime.datetime.now()
date_ = time_.strftime('%Y-%m-%d')
try:
banner = unicode(self.banner, errors='replace')
if self.banner == 'NULL': banner = ''
mongo.NA_INFO.insert({"ip": self.ip, "port": self.port, "hostname": hostname, "banner": banner, "time": time_})
self.statistics[date_]['add'] += 1
ip2hostname 具体数据原理后续补上
请注意： ip2hostname 里的
socket.socket(socket.AF_INET, socket.SOCK_DGRAM) --是指数据报式socket , for UDP
sendto(query_data, (ip, dport)) --将query_data数据发送到套接字，address是形式为（ipaddr，port）的元组，指定远程地址。返回值是发送的字节数。该函数主要用于UDP协议。
.recvfrom(1024) --与recv()类似，但返回值是（data,address）。其中data是包含接收数据的字符串，address是发送数据的套接字地址。
# coding:utf-8 import mongo import log import socket import datetime import urllib2 import re import time import ssl import gzip import StringIO try: _create_unverified_https_context = ssl._create_unverified_context # 忽略证书错误 except AttributeError: pass else: ssl._create_default_https_context = _create_unverified_https_context class scan: def __init__(self, task_host, port_list): self.ip = task_host self.port_list = port_list self.config_ini = {} def run(self): self.timeout = int(self.config_ini['Timeout']) for _port in self.port_list: self.server = '' self.banner = '' self.port = int(_port) self.scan_port() # 端口扫描 if not self.banner:continue self.server_discern() # 服务识别 if self.server == '': web_info = self.try_web() # 尝试web访问 if web_info: log.write('web', self.ip, self.port, web_info) time_ = datetime.datetime.now() mongo.NA_INFO.update({'ip': self.ip, 'port': self.port}, {"$set": {'banner': self.banner, 'server': 'web', 'webinfo': web_info, 'time': time_}}) def scan_port(self): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.connect((self.ip, self.port)) time.sleep(0.2) except Exception, e: return try: self.banner = sock.recv(1024) sock.close() if len(self.banner) <= 2: self.banner = 'NULL' except Exception, e: self.banner = 'NULL' log.write('portscan', self.ip, self.port, None) banner = '' hostname = self.ip2hostname(self.ip) time_ = datetime.datetime.now() date_ = time_.strftime('%Y-%m-%d') try: banner = unicode(self.banner, errors='replace') if self.banner == 'NULL': banner = '' mongo.NA_INFO.insert({"ip": self.ip, "port": self.port, "hostname": hostname, "banner": banner, "time": time_}) self.statistics[date_]['add'] += 1 except: if banner: history_info = mongo.NA_INFO.find_and_modify( query={"ip": self.ip, "port": self.port, "banner": {"$ne": banner}}, remove=True) if history_info: mongo.NA_INFO.insert( {"ip": self.ip, "port": self.port, "hostname": hostname, "banner": banner, "time": time_}) self.statistics[date_]['update'] += 1 del history_info["_id"] history_info['del_time'] = time_ history_info['type'] = 'update' mongo.NA_HISTORY.insert(history_info) def server_discern(self): for mark_info in self.config_ini['Discern_server']: # 快速识别 try: name, default_port, mode, reg = mark_info if mode == 'default': if int(default_port) == self.port: self.server = name elif mode == 'banner': matchObj = re.search(reg, self.banner, re.I | re.M) if matchObj: self.server = name if self.server:break except: continue if not self.server and self.port not in [80,443,8080]: for mark_info in self.config_ini['Discern_server']: # 发包识别 try: name, default_port, mode, reg = mark_info if mode not in ['default','banner']: dis_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) dis_sock.connect((self.ip, self.port)) mode = mode.decode('string_escape') reg = reg.decode('string_escape') dis_sock.send(mode) time.sleep(0.3) dis_recv = dis_sock.recv(1024) matchObj = re.search(reg, dis_recv, re.I | re.M) if matchObj: self.server = name break except: pass if self.server: log.write("server", self.ip, self.port, self.server) mongo.NA_INFO.update({"ip": self.ip, "port": self.port}, {"$set": {"server": self.server}}) def try_web(self): title_str, html = '', '' try: if self.port == 443: info = urllib2.urlopen("https://%s:%s" % (self.ip, self.port), timeout=self.timeout) else: info = urllib2.urlopen("http://%s:%s" % (self.ip, self.port), timeout=self.timeout) html = info.read() header = info.headers except urllib2.HTTPError, e: html = e.read() header = e.headers except: return if not header: return if 'Content-Encoding' in header and 'gzip' in header['Content-Encoding']: # 解压gzip html_data = StringIO.StringIO(html) gz = gzip.GzipFile(fileobj=html_data) html = gz.read() try: html_code = self.get_code(header, html).strip() if html_code and len(html_code) < 12: html = html.decode(html_code).encode('utf-8') except: pass try: title = re.search(r'<title>(.*?)</title>', html, flags=re.I | re.M) if title: title_str = title.group(1) except: pass try: web_banner = str(header) + "\r\n\r\n" + html self.banner = web_banner history_info = mongo.NA_INFO.find_one({"ip": self.ip, "port": self.port}) if 'server' not in history_info: tag = self.get_tag() web_info = {'title': title_str, 'tag': tag} return web_info else: if abs(len(history_info['banner'].encode('utf-8')) - len(web_banner)) > len(web_banner) / 60: del history_info['_id'] history_info['del_time'] = datetime.datetime.now() mongo.NA_HISTORY.insert(history_info) tag = self.get_tag() web_info = {'title': title_str, 'tag': tag} date_ = datetime.datetime.now().strftime('%Y-%m-%d') self.statistics[date_]['update'] += 1 log.write('info', None, 0, '%s:%s update web info'%(self.ip, self.port)) return web_info except: return def ip2hostname(self,ip): try: hostname = socket.gethostbyaddr(ip)[0] return hostname except: pass try: query_data = "\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x20\x43\x4b\x41\x41" + \ "\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41" + \ "\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x00\x00\x21\x00\x01" dport = 137 _s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) _s.sendto(query_data, (ip, dport)) x = _s.recvfrom(1024) tmp = x[0][57:] hostname = tmp.split("\x00", 2)[0].strip() hostname = hostname.split()[0] return hostname except: pass def get_code(self, header, html): try: m = re.search(r'<meta.*?charset=(.*?)"(>| |/)', html, flags=re.I) if m: return m.group(1).replace('"', '') except: pass try: if 'Content-Type' in header: Content_Type = header['Content-Type'] m = re.search(r'.*?charset=(.*?)(;|$)', Content_Type, flags=re.I) if m: return m.group(1) except: pass def get_tag(self): try: url = self.ip + ':' + str(self.port) tag = map(self.discern, ['Discern_cms', 'Discern_con', 'Discern_lang'], [url, url, url]) return filter(None, tag) except Exception, e: return def discern(self, dis_type, domain): file_tmp = {} if int(domain.split(":")[1]) == 443: protocol = "https://" else: protocol = "http://" try: req = urllib2.urlopen(protocol + domain, timeout=self.timeout) header = req.headers html = req.read() except urllib2.HTTPError, e: html = e.read() header = e.headers except Exception, e: return for mark_info in self.config_ini[dis_type]: if mark_info[1] == 'header': try: if not header: return if re.search(mark_info[3], header[mark_info[2]], re.I): return mark_info[0] except Exception, e: continue elif mark_info[1] == 'file': if mark_info[2] == 'index': try: if not html: return if re.search(mark_info[3], html, re.I): return mark_info[0] except Exception, e: continue else: if mark_info[2] in file_tmp: re_html = file_tmp[mark_info[2]] else: try: re_html = urllib2.urlopen(protocol + domain + "/" + mark_info[2], timeout=self.timeout).read() except urllib2.HTTPError, e: re_html = e.read() except Exception, e: return file_tmp[mark_info[2]] = re_html try: if re.search(mark_info[3], re_html, re.I): return mark_info[0] except Exception, e: print mark_info[3]
