python+selenium模拟浏览器抓取智慧芽数据

一、需求

有一批专利号需要在智慧芽上找它对应的相似专利。

抓包看了下智慧芽登录接口，登录可以直接使用用户名、密码登录，没验证码限制少了很多麻烦。又看它使用token请求，开始想着获取到登录后的token再批量请求接口就好了，但尝试了下发现不行。因为请求次数多了就会有限制，再重新登录换新token都不行，所以改成了使用selenium模拟浏览器请求抓取数据。

二、思路

按照人手动操作，先登录，然后打开页面抓取数据，然后保存到excel中，由于数据比较多，代码运行过程中容易出现中断，所以抓取一部分就保存一部分到excel中，一直循环抓取。

三、输入输出格式

输入

输出

四、代码

代码中涉及到的知识点还是比较多的，所以贴了上来，方便后面遇到类似的直接拿来使用。

from selenium import webdriver
import time
import pandas
import pandas as pd
from openpyxl import load_workbook
from selenium.common import NoSuchElementException
from selenium.webdriver.common.by import By
from collections import Counter
from collections import OrderedDict
INPUT_FILE='PNS_1.xlsx'
OUT_FILE='PN_Out_1.xlsx'
# 设置WebDriver
driver = webdriver.Chrome()
# 存放需要保存的数据
save_data=[]
# 登录
def login():
    # 打开Google首页
    driver.get('https://account.zhihuiya.com/#/')
    time.sleep(5)
    current_url=driver.current_url
    print('当前页面url:',current_url)
    if 'account.zhihuiya.com' in current_url:
        tab_element = driver.find_element(By.ID, 'tab-EMAIL_PASSWORD')
        if tab_element.is_displayed():
            tab_element.click()
            time.sleep(3)
            # 找到账号输入框，输入账号
            account_element = driver.find_element(By.XPATH, '//*[@id="pane-EMAIL_PASSWORD"]/form/div[1]/div/input')
            account_element.click()
            account_element.clear()
            account_element.send_keys('xxxx@qq.com')
            pwd_element = driver.find_element(By.XPATH, '//*[@id="pane-EMAIL_PASSWORD"]/form/div[2]/div/div/input')
            pwd_element.send_keys('xxxxx')
            #登录按钮
            btn_element = driver.find_element(By.XPATH, '//*[@id="pane-EMAIL_PASSWORD"]/form/div[4]/div/button')
            btn_element.click()
            time.sleep(5)
            try:
                # 定位元素
                element = driver.find_element(By.XPATH, '/html/body/div[6]/div')
                # 判断元素是否可见
                is_visible = element.is_displayed()
                print(f"元素是否可见: {is_visible}")
                if is_visible:
                    agree_element = driver.find_element(By.XPATH, '/html/body/div[6]/div/div[2]/button[2]')
                    agree_element.click()
                    time.sleep(5)
                # 对元素进行操作
            except NoSuchElementException:
                print("元素不存在")

# 获取PN对应的patentId
def find_patentId(pn):
    url='https://analytics.zhihuiya.com/search/result/tablelist/1?sort=pdesc&limit=100&q='+pn+'&_type=query&search_mode=publication'
    driver.get(url)
    time.sleep(2)
    a_element=driver.find_element(By.XPATH,'//*[@data-link-type="PN"][@data-link-data="'+pn+'"]')
    print(a_element)
    # 获取元素属性
    patent_id = a_element.get_attribute("data-patent-id")
    print(patent_id)
    find_similar(pn, patent_id)

# 根据pn和patentId找相似
def find_similar(pn,patent_Id):
    url='https://analytics.zhihuiya.com/patent-view/similar?_type=query&source_type=search_result&rows=100&patentId='+patent_Id+'&sort=pdesc&page=1&q='+pn
    print(url)
    driver.get(url)
    time.sleep(6)
    span_elements=driver.find_elements(By.XPATH,'//*[@class="PN"]/span[@class="snap-table__pn-row"]')
    print('相似专利:',span_elements)
    global save_data
    for ele in span_elements:
        print(pn,ele.text)
        ele_tup=(pn,ele.text)
        save_data.append(ele_tup)

    # 超过一百条写入excel一次
    if len(save_data)>=100:
        write_diff_PNs()
        print('写入数据')


# 获取已保存到Excel中的PN
def find_out_PNs(sheetname,output_file):
    # 加载工作簿
    workbook = load_workbook(output_file)
    sheet = workbook[sheetname]
    print(sheet)
    rows = [row[0].value for row in sheet.iter_rows(min_col=1, max_col=2, min_row=2)];
    # print(rows)
    workbook.close()
    return  rows

# 获取所有需要获取相似专利的PN
def find_all_PNs(sheetname,output_file):
    # 加载工作簿
    workbook = load_workbook(output_file)
    sheet = workbook[sheetname]
    print(sheet)
    rows = [row[0].value for row in sheet.iter_rows(min_col=1, max_col=1, min_row=2)];
    # print(rows)
    workbook.close()
    return  rows

# 获取所有和已获取PN的差集
def find_diff_PNs():
    out_pns=find_out_PNs('PN',OUT_FILE)
    out_pns=list(set(out_pns))
    print('已存pns个数:',len(out_pns))
    all_pns=find_all_PNs('PN',INPUT_FILE)
    all_pns=list(set(all_pns))
    print('所有pns个数:',len(all_pns))
    diff_pns = [x for x in out_pns if x not in all_pns]
    print('多余pns:',diff_pns)
    diff_pns = [x for x in all_pns if x not in out_pns]
    print('差异pns:',diff_pns)
    return diff_pns

# 找到待获取的PN并将其写入到Excel
def write_diff_PNs():
    # 合并原来旧数据并写入到excel中
    global save_data
    save_data_df = pandas.DataFrame(save_data, columns=['Query', 'Similar'])
    df = pd.read_excel(OUT_FILE, sheet_name='PN')
    print('已存数据行数：',len(df))
    df = pd.concat([df, save_data_df], ignore_index=True)  # 合并数据
    print('删除前数据数：',len(df))
    # drop_duplicates 去除重复数据
    df=df.drop_duplicates(subset=['Query','Similar'],keep="first",ignore_index=True)
    print('删除后数据数：',len(df))
    with pd.ExcelWriter(OUT_FILE, engine='openpyxl', mode='a', if_sheet_exists="replace") as writer:
        df.to_excel(writer, sheet_name='PN', index=False)
    save_data=[]


if __name__ == '__main__':
    # 记录开始时间
    start_time = time.time()
    print(start_time)
    if len(find_diff_PNs())>0:
        login()
        time.sleep(10)
        while len(find_diff_PNs())>0:
            try:
                # 找到待获取的PN
                diff_pns = find_diff_PNs()
                print('差异pns数量',diff_pns)
                for pn in diff_pns:
                    # 找到对应的patentId
                    find_patentId(pn)
                    time.sleep(1)
                time.sleep(10)
                print(save_data)
                write_diff_PNs()
            except Exception:
                print("打印异常")

        time.sleep(10)
        # 关闭浏览器
        driver.quit()
    else:
        print('已无需要查找相似的专利')
        driver.quit()
    # 记录结束时间
    end_time = time.time()
    # 计算耗时
    elapsed_time = end_time - start_time
    print(f"耗时: {elapsed_time} 秒")

    # 统计列表元素重复次数
    # data_list = find_out_PNs('PN',OUT_FILE)
    # counts = Counter(data_list)
    # unique_values = list(OrderedDict.fromkeys(data_list))
    # for unique in unique_values:
    #     value = counts.get(unique)
    #     if value>20:
    #         print(str(unique) + '\t' + str(value))