刘一辰的软件工程随笔

今日目标：代码总结

 

今日总结：

import requests

import pymysql

from bs4 import BeautifulSoup

db = pymysql.connect('127.0.0.1',

                     port=3306,

                     user='root',

                     password='123',

                     db='mytest',

                     charset='utf8')

cursor = db.cursor()

headers={

        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"

        }

url="http://openaccess.thecvf.com/CVPR2019.py"

html=requests.get(url)

soup=BeautifulSoup(html.content,'html.parser')

soup.a.contents=='pdf'

pdfs=soup.findAll(name="a",text="pdf")

lis = []

jianjie=""

for i,pdf in enumerate(pdfs):

    pdf_name=pdf["href"].split('/')[-1]

    name=pdf_name.split('.')[0].replace("_CVPR_2019_paper","")

    link="http://openaccess.thecvf.com/content_CVPR_2019/html/"+name+"_CVPR_2019_paper.html"

    url1=link

    html1 = requests.get(url1)

    soup1 = BeautifulSoup(html1.content, 'html.parser')

    weizhi = soup1.find('div', attrs={'id':'abstract'})

    if weizhi:

        jianjie =weizhi.get_text();

    print("这是第"+str(i)+"条数据")

    keyword = str(name).split('_')

    keywords = ''

    for k in range(len(keyword)):

        if (k == 0):

            keywords += keyword[k]

        else:

            keywords += ',' + keyword[k]

    info = {}

    info['title'] = name

    info['link'] =link

    info['abstract']=jianjie

    info['keywords']=keywords

    lis.append(info)

cursor = db.cursor()

for i in range(len(lis)):

    cols = ", ".join('`{}`'.format(k) for k in lis[i].keys())

    print(cols)  # '`name`, `age`'

    val_cols = ', '.join('%({})s'.format(k) for k in lis[i].keys())

    print(val_cols)  # '%(name)s, %(age)s'

    sql = "insert into lunwen(%s) values(%s)"

    res_sql = sql % (cols, val_cols)

    print(res_sql)

    cursor.execute(res_sql, lis[i])  # 将字典a传入

    db.commit()

    num=1

    print(num)

    print("成功")

　二、分析、查找关键词

 借助Map存储关键词， key为关键词，value为出现的次数。遍历到相同的关键词value值+1，然后根据value值排序。

dao层：

package dao;

import java.sql.Connection;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.LinkedHashMap;

import java.util.Map;

import java.util.stream.Collectors;

import  Bean.copy.*;

import jdbc.Util;

import java.sql.Connection;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

import java.util.ArrayList;

import java.util.List;

import com.sun.xml.internal.ws.policy.privateutil.PolicyUtils.Collections;

public class Dao {

    public static Map<String,Integer> getrc()

    {

        String sql="select * from lunwen";

        Map<String, Integer>map= new HashMap<String, Integer>();

        Map<String, Integer>results= new LinkedHashMap<String, Integer>();

        Connection con=null;

        Statement state=null;

        ResultSet rs=null;

        con=Util.getConn();

        try {

            state=con.createStatement();

            rs=state.executeQuery(sql);

            while(rs.next())

            {

                String keywords=rs.getString("keywords");

                String[] split = keywords.split(",");

                for(int i=0;i<split.length;i++)

                {

                    if(map.get(split[i])==null)

                    {

                        map.put(split[i],0);

                    }

                    else

                    {

                        map.replace(split[i], map.get(split[i])+1);

                    }

                }

            }

        } catch (SQLException e) {

            // TODO Auto-generated catch block

            e.printStackTrace();

        }

        Util.close(rs, state, con);

        map.entrySet()                

        .stream()               

        .sorted((p1, p2) -> p2.getValue().compareTo(p1.getValue()))                

        .collect(Collectors.toList())

        .forEach(ele -> results.put(ele.getKey(), ele.getValue()));

        return results;

    }

    public List<Data> list(String keywords) { // 查询所有信息

        List<Data> list = new ArrayList<Data>(); // 创建集合

        Connection conn = Util.getConn();

        String sql = "select * from lunwen where keywords like "+"'%"+keywords+"%'"; // SQL查询语句

        try {

            PreparedStatement pst = conn.prepareStatement(sql);

            ResultSet rs = pst.executeQuery();

            Data data = null;

            while (rs.next()) {

                String title = rs.getString("title");

                String link = rs.getString("link");

                String as= rs.getString("abstract");

                    data = new Data(title,link,as,keywords);

                list.add(data);

            }

            rs.close(); // 关闭

            pst.close(); // 关闭

        } catch (SQLException e1) {

            e1.printStackTrace(); // 抛出异常

        }

        return list; // 返回一个集合

    }

}

　　servlet层：

package servlet;

import java.io.IOException;

import java.util.Map;

import javax.servlet.ServletException;

import javax.servlet.annotation.WebServlet;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import dao.Dao;

import net.sf.json.JSONArray;

import net.sf.json.JSONObject;

@WebServlet("/RcServlet")

public class RcServlet extends HttpServlet {

    private static final long serialVersionUID = 1L;

    /**

     * @see HttpServlet#HttpServlet()

     */

    public RcServlet() {

        super();

        // TODO Auto-generated constructor stub

    }

    /**

     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        this.doPost(request, response);

    }

    /**

     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)

     */

    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        request.setCharacterEncoding("utf-8");

        Map<String, Integer>sortMap=Dao.getrc();

        JSONArray json =new JSONArray();

        int k=0;

        for (Map.Entry<String, Integer> entry : sortMap.entrySet()) 

        {

            JSONObject ob=new JSONObject();

            ob.put("name", entry.getKey());

            ob.put("value", entry.getValue());

            if(!(entry.getKey().equals("for")||entry.getKey().equals("and")||entry.getKey().equals("With")||entry.getKey().equals("of")||entry.getKey().equals("in")||entry.getKey().equals("From")||entry.getKey().equals("A")||entry.getKey().equals("to")||entry.getKey().equals("a")||entry.getKey().equals("the")||entry.getKey().equals("by")))

            {

                json.add(ob);

                k++;

            }

            if(k==10)

                break;

        }

        System.out.println(json.toString());

        response.getWriter().write(json.toString());

    }

}

　　三、生成热词汇云图

<%@ page language="java" contentType="text/html; charset=UTF-8"

    pageEncoding="UTF-8"%>

<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>

<%request.setCharacterEncoding("utf-8"); 

response.setCharacterEncoding("utf-8");%>

<!DOCTYPE html>

<html>

<head>

<meta charset="ISO-8859-1">

<title>热词云</title>

</head>

<meta charset="UTF-8">

     <link type="text/css" rel="stylesheet" href="css/style.css">

<script src="js/jquery-3.4.1.min.js"></script>

<script src="js/echarts.min.js"></script>

<script src="js/echarts-cloud.js"></script>

<style>

           #main{

              width: 30%;

              height: 500px;

              border:1px solid #ddd;

              float:right;

          }

          #table{

                overflow-x: auto;

                 overflow-y: auto;

                width: 70%;

                height: 500px;

                float:left;

                margin-top:100dp;

                padding-top:100dp;

            }

        </style>

</head>

<body >

<br>

<h1>热词云</h1>

<br>

<br>

<br>

<div id="table">

  <table id='gradient-style' >

    <tr>

      <th align="center">论文连接</th>

    </tr>

    <c:forEach var="item" items="${list}">

      <tr>

        <td><a href="${item.link}">${item.title}</a></td>

      </tr>

    </c:forEach>

  </table>

</div>

  <div id="main">

  </div>

  <script type="text/javascript">

    var dt;

            $.ajax({

                url : "RcServlet",

                async : true,

                type : "POST",

                data : {        

                },

                dataType : "json",

                success : function(data) {

                    dt = data;

                     var mydata = new Array(0);

                     for (var i = 0; i < dt.length; i++) {

                          var d = {};

                          d["name"] = dt[i].name;

                          d["value"] = dt[i].value;

                          mydata.push(d);

                      }

                     var myChart = echarts.init(document.getElementById('main'));

                     //设置点击效果

                     myChart.setOption({

                         title: {

                             text: ''

                         },

                         tooltip: {},

                         series: [{

                             type : 'wordCloud',  //类型为字符云

                                 shape:'smooth',  //平滑

                                 gridSize : 8, //网格尺寸

                                 size : ['50%','50%'],

                                 //sizeRange : [ 50, 100 ],

                                 rotationRange : [-45, 0, 45, 90], //旋转范围

                                 textStyle : {

                                     normal : {

                                         fontFamily:'微软雅黑',

                                         color: function() {

                                             return 'rgb(' + 

                                                 Math.round(Math.random() * 255) +

                                          ', ' + Math.round(Math.random() * 255) +

                                          ', ' + Math.round(Math.random() * 255) + ')'

                                                }

                                         },

                                     emphasis : {

                                         shadowBlur : 5,  //阴影距离

                                         shadowColor : '#333'  //阴影颜色

                                     }

                                 },

                                 left: 'center',

                                 top: 'center',

                                 right: null,

                                 bottom: null,

                                 width:'100%',

                                 height:'100%',

                                 data:mydata

                         }]

                     });

                     myChart.on('click', function (params) {

                         var url = "ClickServlet?keywords=" + params.name;

                         window.location.href = url;

                       });

                    alert("成功！");

                },

                error : function() {

                    alert("请求失败");

                },

           });

 

</script>

</body>

结果：

</html>

 

TRANSLATE with x

English

Arabic	Hebrew	Polish
Bulgarian	Hindi	Portuguese
Catalan	Hmong Daw	Romanian
Chinese Simplified	Hungarian	Russian
Chinese Traditional	Indonesian	Slovak
Czech	Italian	Slovenian
Danish	Japanese	Spanish
Dutch	Klingon	Swedish
English	Korean	Thai
Estonian	Latvian	Turkish
Finnish	Lithuanian	Ukrainian
French	Malay	Urdu
German	Maltese	Vietnamese
Greek	Norwegian	Welsh
Haitian Creole	Persian

 

TRANSLATE with

COPY THE URL BELOW

Back

EMBED THE SNIPPET BELOW IN YOUR SITE

Enable collaborative features and customize widget: Bing Webmaster Portal

Back