[开发参考/Python]Python爬虫框架

本文章的源代码来源于https://github.com/Holit/Web-Crawler-Framwork

一、爬虫框架的代码

import urllib.request
from bs4 import BeautifulSoup
import re
import time
import _thread

# Input your Url here####################################
BaseURL = '127.0.0.1/'
#########################################################
TaxURL = ".html"

#Input your data-saving path ############################
SavePath = ""
#########################################################

#Input your threads count ###############################
thread_count = 1
#########################################################

#Set each spider will spy how many pages ################
thread_spy_count_ench = 5
#########################################################
def mkdir(path):
    # Create the directory
    import os
    path=path.strip()
    path=path.rstrip("\\")
    isExists=os.path.exists(path)
    if not isExists:
        os.makedirs(path)
        return True
    else:
        return False

def download(start, count):
    #Spider main
    for i in range(start,start + count):
        try:
            #DEBUG##################################################
            #print("[INFO] Connecting to page #" + str(i) + "...")
            ########################################################
            
            #Used to record time
            time_start=time.time()
            
            #Construct url
            #This only work like
            # https://127.0.0.1/articles/00001.html
            # https://127.0.0.1/articles/00002.html
            # https://127.0.0.1/articles/00003.html
            TargetURL = BaseURL + str(i) + TaxURL
            
            
            #create Request object
            req = urllib.request.Request(TargetURL)
            #create headers using general header, you could find this by Fiddler(R) or by Chrome(R)
            req.add_header('Host','')    #Your Host, usally set as url-base
            req.add_header('Referer',TargetURL)        #Your Referer, usally set as url
            req.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19')
            #finishing create Request object
            
            #get information
            res = urllib.request.urlopen(req)
            #decode the html
            soup = BeautifulSoup(res,"html.parser")
            ##############################################################
            #add your functions  here....
            
            #operate_data(data)
            
            #soup find div to get inforamtion.
            #soup is able to opreate html tag very easily, by using soup.find(...)
            ##############################################################
            
            #Change saving path here.
            savetarget = SavePath
            
            #trying to saving files
            try:
                #create directory if it doesn't existed
                mkdir(SavePath+"\\"+str(zone)+"\\")
                #using open...
                f = open(savetarget,'w')
                
                #edit this
                f.write("data")
                
            except Exception as e:
                time_end=time.time()
                print("  [Failed] - #" + str(i) + " Error : " + str(e))
            else:
            
                time_end=time.time()
                print("  [Succeed] - #" + str(i) + " has saved to path.("+str(time_end-time_start)+"s)")

            pass
        except Exception as e:
            print("  [Global Failure] - #" + str(i) + " Error : " + str(e))
            pass



#if __name__ == __main__:
try:
    #Multithreading
    print("Spidering webiste...")
    print("Current configuration :")
    print("--Will create " + str(thread_count) + "threads to access.")
    print("--Will save to " + SavePath)
    print("-------------START---------------------------")
    # press any key to continue
    # this won't work under linux
    import os
    os.system('pause')
    try:
        for i in range(0,thread_count):
            print("[Thread #"+ str (i) +"] started successfully")
            _thread.start_new_thread(download, (thread_spy_count_ench * i,thread_spy_count_ench))
    except Exception as e:
        print("[Threading@" + str(i) +"] Error:"+ str(e))
except Exception as e:
   print("[Global Failure] Error:"+ str(e))
while 1:
   pass

二、对其中功能的实例化操作

　　1.文本获取功能

　　　　文本获取是指对页面的<div class='content'>...</div>中的内容进行获取，这是前提。如果不同需要更改。

　　　　（1）思路

　　　　　　使用BeautifulSoup对html分析之后得到解码的文件，例如

            <div class="content" style="text-align: left">
            基础内容
            </div>

　　　　　　现在对该段落进行选取，即使用soup.find功能

　　　　（2）基本代码

passages_div = soup.find('div')
passages_set = passages_div.findAll(attrs={"class":"content"})
for passages in passages_set:
    article = str(passages)
    #文字处理
    article = article.replace('<div class="content" style="text-align: left">', '')
    article = article.replace(u'\ue505', u' ')#对Unicode的空格进行处理，如果不处理gbk无法编码
    article = article.replace(u'\ue4c6', u' ')
    article = article.replace(u'\xa0', u' ')
    article = article.replace('<br/>', '\n')
    article = article.replace('</div>', '')
    savetarget = 'D:\test\test.txt'
    try:
        mkdir('D:\test\')
        f = open(savetarget,'w')
        f.write(article)
    except Exception as e:
        print("  [Failed] - "+ str(e))
    else:
        time_end=time.time()
        print("  [Succeed] - saved to path.")

pass

　　2.图片获取操作

　　　　图片获取一般是通过对网页上的<img src="127.0.0.1/png.png">Hello</img>中src上的内容进行下载操作

　　　　目前可以使用多种操作方式，例如urlretrieve，不再赘述