Python 2.7.12从腾讯企业邮箱爬取简历

由于公司需要做一个hrm的项目，这是其中简历拉取的一个功能。本来应该写一个自动发布的但是由于各大平台没有api使得无果。

拉取的网站主要是猎聘和前程无忧，公司没有用boss直聘，而实习僧是必须扫码登陆的，拉勾则是由于gbk与utf-8的转解码问题没有拉取成功

这里参考了部分网上的代码https://www.jb51.net/article/120269.htm，略作修改并加上了部分注释

#!/usr/bin/python2.7
# _*_ coding: utf-8 _*_

"""
@Author: MarkLiu
"""

import poplib
poplib._MAXLINE=20480
import email
import re
from email.parser import Parser
from email.header import decode_header
from email.utils import parseaddr

def decode_str(s):
    value, charset = decode_header(s)[0]
    if charset:
        value = value.decode(charset)
    return value

def guess_charset(msg):
    # 先从msg对象获取编码:
    charset = msg.get_charset()
    if charset is None:
        # 如果获取不到，再从Content-Type字段获取:
        content_type = msg.get('Content-Type', '').lower()
        pos = content_type.find('charset=')
        if pos >= 0:
            charset = content_type[pos + 8:].strip()
    return charset


def get_email_headers(msg):
    # 邮件的From, To, Subject存在于根对象上:
    headers = {}
    for header in ['From', 'To', 'Subject', 'Date']:
        value = msg.get(header, '')
        if value:
            if header == 'Date':
                headers['date'] = value
            if header == 'Subject':
                # 需要解码Subject字符串:
                subject = decode_str(value)
                headers['subject'] = subject
            else:
                # 需要解码Email地址:
                hdr, addr = parseaddr(value)
                name = decode_str(hdr)
                value = u'%s <%s>' % (name, addr)
                if header == 'From':
                    from_address = value
                    headers['from'] = from_address
                else:
                    to_address = value
                    headers['to'] = to_address
    content_type = msg.get_content_type()
    print 'head content_type: ', content_type
    return headers

att_file = "";
# indent用于缩进显示:
def get_email_cntent(message, base_save_path):
    j = 0
    content = ''
    attachment_files = []
    for part in message.walk():
        j = j + 1
        file_name = part.get_filename()
        contentType = part.get_content_type()
        # 保存附件
        if file_name:  # Attachment
            h = email.Header.Header(file_name)
            dh = email.Header.decode_header(h)
            name = get_email_headers(msg)['from']
            filename = re.findall('.*51job] (.*) <resume.*', name) # 这里是只有前程无忧有附件
            if dh[0][1]:  # 如果包含编码的格式，则按照该格式解码
                filename = unicode(filename[0], dh[0][1])
                filename = filename[0].encode("utf-8")
            data = part.get_payload(decode=True)
            att_file = open(base_save_path + filename[0] + ".html", 'wb') # 将默认的附件名resume改成了 "new_resume_人名"

            attachment_files.append(filename[0] + ".html")
            att_file.write(data)
            att_file.close()
        elif contentType == 'text/plain' or contentType == 'text/html':
            # 保存正文
            data = part.get_payload(decode=True)
            charset = guess_charset(part)
            if charset:
                charset = charset.strip().split(';')[0]
                print 'charset:', charset
                data = data.decode(charset, "ignore")
            content = data
    return content, attachment_files

if __name__ == '__main__':
    # 输入邮件地址, 口令和POP3服务器地址:
    emailaddress = '****@***.com'
    # 注意使用开通POP，SMTP等的授权码
    password = '*********' # 邮箱密码
    pop3_server = 'smtp.exmail.qq.com'

    # 连接到POP3服务器:
    server = poplib.POP3_SSL(pop3_server,port=995)
    # 可以打开或关闭调试信息:
    # server.set_debuglevel(1)
    # POP3服务器的欢迎文字:
    print server.getwelcome()
    # 身份认证:
    server.user(emailaddress)
    server.pass_(password)
    # stat()返回邮件数量和占用空间:
    messagesCount, messagesSize = server.stat()
    print 'messagesCount:', messagesCount
    print 'messagesSize:', messagesSize
    # list()返回所有邮件的编号:
    resp, mails, octets = server.list()
    print '------ resp ------'
    print resp  # +OK 46 964346 响应的状态 邮件数量 邮件占用的空间大小
    print '------ mails ------'
    print mails  # 所有邮件的编号及大小的编号list，['1 2211', '2 29908', ...]
    print '------ octets ------'
    print octets

    # 获取最新的邮件，一共是看20封，但是会删掉拉钩和实习僧，所以不足20封
    length = len(mails)-1
    for i in range(length):
        if i < 20:
            resp, lines, octets = server.retr(length-i+1)
            print resp
            print lines 
            print octets
            # lines存储了邮件的原始文本的每一行,
            # 可以获得整个邮件的原始文本:
            msg_content = '\n'.join(lines)
            # 把邮件内容解析为Message对象：
            msg = Parser().parsestr(msg_content)
            address = get_email_headers(msg)['from']
            if address[0:3] == u'拉勾网':
                continue
            elif address[0:3] == u'实习僧':
                continue
            # 但是这个Message对象本身可能是一个MIMEMultipart对象，即包含嵌套的其他MIMEBase对象，
            # 嵌套可能还不止一层。所以我们要递归地打印出Message对象的层次结构：
            print u'---------- 解析之后 ----------'
            base_save_path = r'static\common_static\resume\new_resume_'
            msg_headers = get_email_headers(msg)
            print base_save_path
            content, attachment_files = get_email_cntent(msg, base_save_path)

            print 'subject                  :', msg_headers['subject'] # 邮件标题
            print 'from_address             :', msg_headers['from'] # 邮件来源
            print 'to_address               :', msg_headers['to'] # 这个应该是收信邮箱，我也没搞懂，一个32一个48一个tue一个web
            print 'date                     :', msg_headers['date'] # 收到时间
            print 'content                  :', content # html
            print 'attachment_files         :', attachment_files # 附件名称

    # 关闭连接:
    server.quit()