切割web日志
log='''10.1.1.95 - e800 [18/Mar/2005:12:21:42 +0800] \
"GET /stats/awstats.pl?config=e800 HTTP/1.1" 200 899 "http://10.1.1.1/pv/" \
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon)"'''
def jpm(b:str)->list:
lst=[]
flag=False
for word in log.split():
if not flag and (word.startswith('[') or word.startswith('"')):
if word.endswith(']') or word.endswith('"'):
lst.append(word.strip('[]"'))
continue
flag=True
tmp=word[1:]
continue
if flag:
if word.endswith(']') or word.endswith('"'):
tmp+=' '+word[:-1] # tmp+=word.strip(']"')
lst.append(tmp)
flag=False
# continue # continue提取
else:
tmp+=' '+word
# continue # continue 提取
continue
lst.append(word)
return lst
print(jpm(log))
def dip(sub:str)->list:
lst=[]
flag=False
for word in log.split():
# 判断flag提高效率
if not flag and (word.startswith('[') or word.startswith('"')):
tmp=word
if tmp[-1] == ']' or tmp[-1] == '"': # 对"-"特殊情况做处理
tmp=tmp.strip('[]"')
lst.append(tmp)
continue
flag=True
continue
if flag: # 进入特殊处理
if word.endswith(']') or word.endswith('"'):
tmp+=' '+word
flag=False
# print(tmp)
tmp=tmp.strip('[]"')
lst.append(tmp)
continue
else:
tmp+=' '+word
continue
lst.append(word)
return lst
print(dip(log))
import datetime
log='''10.1.1.95 - e800 [18/Mar/2005:12:21:42 +0800] \
"GET /stats/awstats.pl?config=e800 HTTP/1.1" 200 899 "http://10.1.1.1/pv/" \
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon)"'''
def convert_time(timestr:str)->datetime.datetime:
fmtstr='%d/%b/%Y:%H:%M:%S %z'
dt=datetime.datetime.strptime(timestr,fmtstr)
print(dt,type(dt))
return dt
def convert_request(request:str)->dict:
return dict(zip(('method','url','protocol'),request.split()))
def jix(b:str)->list:
lst=[]
flag=False
for word in log.split():
if not flag:
if word[0] == '[' or word.startswith('"'):
if word[-1] == ']' or word.endswith('"'):
lst.append(word[1:-1])
else:
flag=True
tmp=word[1:]
else:
lst.append(word)
continue
if flag:
if word[-1] == ']' or word.endswith('"'):
lst.append(tmp+' '+word[:-1])
flag=False
else:
tmp+=' '+word
continue
return lst
print(jix(log))
def pim():
names = ['remote', 'logname', 'username', 'datetime',
'request', 'status', 'size', 'referer', 'useragent']
funcs = (None, None, None, convert_time,
convert_request, int, int, None, None)
d = {}
for i, field in enumerate(jix(log)):
if funcs[i] is not None:
d.setdefault(names[i], funcs[i](field))
else:
d.setdefault(names[i], field)
return d
print(pim())
import re,datetime
log='''10.1.1.95 - e800 [18/Mar/2005:12:21:42 +0800] \
"GET /stats/awstats.pl?config=e800 HTTP/1.1" 200 899 "http://10.1.1.1/pv/" \
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon)"'''
def extract(line):
pattern='''(?P<remote>[\d\.]{7,})(?= ) (?P<logname>[\w-]+) (?P<username>[\w-]+) \[(?P<time>[^][]+)\] "(?P<request>[^"]+)" (?P<status>\d{3}) (?P<size>\d+) "(?P<referer>[^"]+)" "(?P<useragent>[^"]+)"'''
regex=re.compile(pattern,flags=re.S|re.I)
matcher=regex.match(log)
return matcher.groupdict()
print(extract(log))
def convert_time(timestr:str)->datetime.datetime:
fmtstr='%d/%b/%Y:%H:%M:%S %z'
return datetime.datetime.strptime(timestr,fmtstr)
def convert_request(request:str)->dict:
return dict(zip(('method','url','protocol'),request))
funcs={
'time':convert_time,
# 'time': lambda timestr:datetime.datetime.strptime(timestr,'%d/%b/%Y:%H:%M:%S %z')
'status':int,
'size':int,
'request':convert_request
# 'request': lambda request:dict(zip(('method','url','protocol'),request))
}
ciz={}
for k,v in extract(log).items():
ciz[k]=funcs.get(k,lambda m:m)(v) # default为函数,调用返回v
# print(k,v)
# if funcs.get(k,None):
# ciz[k]=funcs[k](v)
# else:
# ciz.update(((k,v),))
print(ciz)
bb={k:funcs.get(k,lambda m:m)(v) for k,v in extract(log).items()}
print(bb)
import re,datetime
log='''10.1.1.95 - e800 [18/Mar/2005:12:21:42 +0800] \
"GET /stats/awstats.pl?config=e800 HTTP/1.1" 200 899 "http://10.1.1.1/pv/" \
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon)"'''
pattern='''(?P<remote>[\d\.]{7,})(?= ) (?P<logname>[\w-]+) (?P<username>[\w-]+) \[(?P<time>[^][]+)\] "(?P<request>[^"]+)" (?P<status>\d{3}) ('''
regex=re.compile(pattern,flags=re.M|re.I)
def extract(line):
matcher=regex.match(line)
# if matcher:
# return (k:funcs.get(k,lambda m:m)(v) for k,v in matcher.groupdict().items())
# else:
# raise Exception('None match')
if matcher: # matcher 为None,返回默认None
return (k:funcs.get(k,lambda m:m)(v) for k,v in matcher.groupdict().items())
funcs={
'time':convert_time,
# 'time': lambda timestr:datetime.datetime.strptime(timestr,'%d/%b/%Y:%H:%M:%S %z')
'status':int,
'size':int,
'request':convert_requesta# 'request': lambda request:dict(zip(('method','url','protocol'),request))
}
lines=[]
for line in lines:
try:
d=extract(line)
except:
pass
for line in lines:
d=extract(line)
if d:
pass
else:
# todo 统计不合格日志
pass
def load(path):
with open(path,mode='rt+',encoding='utf8') as f:
for line in f:
fields=extract(line)
if fields:
yield fields
else:
pass # todo
def extract(sub):
matcher=regex.search(sub)
if matcher:
return matcher.group()
else:
raise Exception('None match')
print(extract(n))
def extract(line)->dict: # 不匹配,抛异常
matcher=regex.match(line)
if matcher:
return {k:funcs.get(k,lambda m:m)(v) for k,v in matcher.groupdict().items()}
else:
raise Exception("None match")
def extract(line)->dict: # 不匹配,返回None
matcher=regex.match(line)
niz=None
if matcher:
niz={k:funcs.get(k,lambda m:m)(v) for k,v in matcher.groupdict().items()}
return niz