对数据html文本 的处理
对数据html文本 的处理 :
提取文字、图片、分句
''' SELECT * FROM Info_Roles WHERE Flag=1 LIMIT 2; select top y * from 表 where 主键 not in(select top (x-1)*y 主键 from 表) 如果表中无主键,可以用临时表,加标识字段解决.这里的x,y可以用变量. select id=identity(int,1,1),* into #tb from 表 select * from #tb where id between (x-1)*y and x*y-1 select top 1000 Info_ID from Info_Roles select top 2000 Info_ID,',xiaole20180410SPLIT,',content from Info_Content where Info_ID not in( select top 1000 Info_ID from Info_Roles ) ; select top 399 Info_ID,',xiaole20180410SPLIT,',UPPER(content) from Info_Content ; select top 399 CHARINDEX('IMG',UPPER(content)) from Info_Content ; select top 15 Info_ID,',xiaole20180410SPLIT,',content from Info_Content where CHARINDEX('IMG',UPPER(content))>0; select top 15 Info_ID,',xiaole20180410SPLIT,',content from Info_Content where Info_ID in( select top 1000 Info_ID from Info_Roles ) and CHARINDEX('IMG',UPPER(content))>0 ; SELECT TOP 15 Info_ID, ',xiaole20180410SPLIT,', content FROM Info_Content WHERE Info_ID IN ( SELECT TOP 1000 Info_ID FROM Info_Roles WHERE Flag = 1 ) AND CHARINDEX('IMG', UPPER(content)) > 0; SELECT TOP 200 Info_ID, ',xiaole20180410SPLIT,', content FROM Info_Content WHERE Info_ID IN ( SELECT TOP 90000 Info_ID FROM Info_Roles ) AND CHARINDEX('<IMG', UPPER(content)) > 0; ''' from bs4 import BeautifulSoup from selenium import webdriver xlsplit_str = ',xiaole20180410SPLIT,' f_db_txt, uid_d = 'db.uid.para.txt', {} with open(f_db_txt, 'r', encoding='utf-8') as fr: for i in fr: i = i.replace('\t', '').replace('\n', '') if xlsplit_str in i: l = i.split(xlsplit_str) uid = l[0].replace(' ', '') uid_d[uid] = {} uid_d[uid]['html'] = [] uid_d[uid]['html'].append(l[1]) else: uid_d[uid]['html'].append(i) r_d = {} for uid in uid_d: str_ = ''.join(uid_d[uid]['html']) fhtml = 'qqzong.vedio.allinone.tmp.html' with open(fhtml, 'w', encoding='utf-8') as fw: fw.write(str_) with open(fhtml, 'r', encoding='utf-8') as fo: soup = BeautifulSoup(fo, 'html.parser') img_l = soup.find_all('img') if len(img_l) > 0: l = soup.find_all('img') uid_d[uid]['img'], uid_d[uid]['txt'] = [i.attrs['src'] for i in l], soup.text.replace(' ','').replace(' ','').replace(' ','') r_d[uid] = {} r_d[uid] = uid_d[uid] incr_l = ['http://www.51g3.net/templates/images/logo.jpg', 'http://www.51g3.net/attached/image/20171206104541_20247.jpg', 'http://www.51g3.net/attached/image/20171129183441_78749.png', 'http://www.51g3.net/templates/images/agentimg.jpg'] r_d[uid]['img'] += incr_l else: print(uid) pass