html内容为table解析

#html
#内容解析
from lxml import etree
import pandas as pd
import re
def get_data(html):
    html = etree.HTML(html)
    #result = etree.tostring(html)
    #print(result)
    table = html.xpath('//table[contains(@class,"torrents")]')
    # table内容转换成dataframe数据
    result =[]
    for rows in table[0].xpath('./tr'): #备注因为table有表头，所以从第一行开始抓取数据
        #去掉标题行
        elements = rows.xpath('./td')
        tmp_result = [''.join(ele.xpath('.//text()') ) for ele in elements]
        title_id = rows.xpath(".//a[contains(@href,'php?id')]/@href")
        try:
            title_id = re.search('php\?id=(\d+)',str(title_id)).group(1)    
        except:
            title_id = ''
        #title_class  = ''.join(rows.xpath(".//a[contains(@href,'?cat=')]/img/@title"))
        #print(rows.xpath(".//a[contains(@href,'php?id')]/@title"))
        tmp_result.append(title_id)
        #tmp_result.append(title_class)    

        result.append(tmp_result)

    labels = ['类型','标题','评论数','存活时间','大小','种子数','下载数','完成数','发布者','标题id']
    result
    df = pd.DataFrame.from_records(result,columns=labels)
    #删除第一行数据
    df=df.iloc[1:]
    return df
#get_data(html)

存入mongod数据库

import pandas as pd
from pymongo import MongoClient
from sqlalchemy import create_engine
def data_to_dataframe(data):
    from pymongo import MongoClient
    client = MongoClient('67.216.204.220', 27017)
    db = client.pt
    table = db.pt_btschool_net_torrents
    records = data.to_dict('records')
    table.insert_many(records)
    return 'success dataframe_to_mongodb '
#data_to_dataframe(df)

html 内容是table的解析办法：

说明：table中tbody是无效的，使用中不需要用它

import pandas as pd
from lxml import html

url = "http://www.uesp.net/wiki/Skyrim:No_Stone_Unturned"
xpath = "//*[@id=\"mw-content-text\"]/table[3]"

tree = html.parse(url)
table = tree.xpath(xpath)[0]
raw_html = html.tostring(table)

dta = pd.read_html(raw_html, header=0)[0]
dta["completed"] = 0
del dta["Map"]
参考地址：https://gist.github.com/jseabold/5892603

案例：

from lxml import etree
import pandas as pd
table = html.xpath('//table[@id="torrenttable"]')[0]
raw_html = etree.tostring(table)
data = pd.read_html(raw_html, header=0)[0]

读取mongodb数据内容

import pandas as pd
from sqlalchemy import create_engine
def read_mongb():
    from pymongo import MongoClient
    client = MongoClient('67.216.204.220', 27017)
    db = client.pt
    table = db.pt_btschool_net_torrents
    data = pd.DataFrame(pd.DataFrame(list(table.find())))
    return data
data = read_mongb()
data.head()
data.columns