热词统计

DAO.java

package dao;

import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

import entity.Cvf;
import utils.DBUtil;

public class dao {
	//添加数据入库
		public boolean add(Cvf cvf) {
			String sql="insert into cvpr(cname,chref,cabstract,ckeyword) values (?,?,?,?)";
			Object obj[]= {cvf.getCname(),cvf.getChref(),cvf.getCabstract(),cvf.getCkeyword()};
			return DBUtil.executeUpdate(sql, obj);
		}
		//查询数据
		 public List<Cvf> Query() {
	  	   List<Cvf> cvfs=new ArrayList();
	  	   Cvf cvf= null;
		       ResultSet rs = null; 
	  	   try {
		    	 String sql="select * from cvpr  " ;
		    	 Object [] params= {};
		    	 rs=DBUtil.executeQuery(sql, params);
		         while(rs.next()) {
		        	 int Id=rs.getInt("id");
		        	 String cname=rs.getString("cname");
		        	 String chref=rs.getString("chref");
		        	 String cabstract=rs.getString("cabstract");
		        	 String ckeyword=rs.getString("ckeyword");
		        	 cvf=new Cvf(Id,cname,chref,cabstract,ckeyword);
		        	 cvfs.add(cvf);
		         }
		     }catch(SQLException e) {
		    	 e.printStackTrace();
		     }catch(Exception e) {
		    	 e.printStackTrace();
		     }finally {
		    	 try {
	     			//先开的后关，后开的先关
	     		if(rs!=null)rs.close();
	     		if(DBUtil.pstmt!=null)DBUtil.pstmt.close();
	     		if(DBUtil.connection !=null)DBUtil.connection.close();
	     		}catch(SQLException e) {
	     			e.printStackTrace();
	     		}finally {
	     			
	     		}
		    	
		     }
	  	   return cvfs;
	     }
		//查询数据
		 public List<Cvf> Query(String key) {
	  	   List<Cvf> cvfs=new ArrayList();
	  	   Cvf cvf= null;
		       ResultSet rs = null; 
	  	   try {
		    	 String sql="select * from cvpr where ckeyword=? " ;
		    	 Object [] params= {key};
		    	 rs=DBUtil.executeQuery(sql, params);
		         while(rs.next()) {
		        	 int Id=rs.getInt("id");
		        	 String cname=rs.getString("cname");
		        	 String chref=rs.getString("chref");
		        	 String cabstract=rs.getString("cabstract");
		        	 cvf=new Cvf(Id,cname,chref,cabstract,key);
		        	 cvfs.add(cvf);
		         }
		     }catch(SQLException e) {
		    	 e.printStackTrace();
		     }catch(Exception e) {
		    	 e.printStackTrace();
		     }finally {
		    	 try {
	     			//先开的后关，后开的先关
	     		if(rs!=null)rs.close();
	     		if(DBUtil.pstmt!=null)DBUtil.pstmt.close();
	     		if(DBUtil.connection !=null)DBUtil.connection.close();
	     		}catch(SQLException e) {
	     			e.printStackTrace();
	     		}finally {
	     			
	     		}
		    	
		     }
	  	   return cvfs;
	     }
}

　　Cvf.java

package dao;

import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

import entity.Cvf;
import utils.DBUtil;

public class dao {
	//添加数据入库
		public boolean add(Cvf cvf) {
			String sql="insert into cvpr(cname,chref,cabstract,ckeyword) values (?,?,?,?)";
			Object obj[]= {cvf.getCname(),cvf.getChref(),cvf.getCabstract(),cvf.getCkeyword()};
			return DBUtil.executeUpdate(sql, obj);
		}
		//查询数据
		 public List<Cvf> Query() {
	  	   List<Cvf> cvfs=new ArrayList();
	  	   Cvf cvf= null;
		       ResultSet rs = null; 
	  	   try {
		    	 String sql="select * from cvpr  " ;
		    	 Object [] params= {};
		    	 rs=DBUtil.executeQuery(sql, params);
		         while(rs.next()) {
		        	 int Id=rs.getInt("id");
		        	 String cname=rs.getString("cname");
		        	 String chref=rs.getString("chref");
		        	 String cabstract=rs.getString("cabstract");
		        	 String ckeyword=rs.getString("ckeyword");
		        	 cvf=new Cvf(Id,cname,chref,cabstract,ckeyword);
		        	 cvfs.add(cvf);
		         }
		     }catch(SQLException e) {
		    	 e.printStackTrace();
		     }catch(Exception e) {
		    	 e.printStackTrace();
		     }finally {
		    	 try {
	     			//先开的后关，后开的先关
	     		if(rs!=null)rs.close();
	     		if(DBUtil.pstmt!=null)DBUtil.pstmt.close();
	     		if(DBUtil.connection !=null)DBUtil.connection.close();
	     		}catch(SQLException e) {
	     			e.printStackTrace();
	     		}finally {
	     			
	     		}
		    	
		     }
	  	   return cvfs;
	     }
		//查询数据
		 public List<Cvf> Query(String key) {
	  	   List<Cvf> cvfs=new ArrayList();
	  	   Cvf cvf= null;
		       ResultSet rs = null; 
	  	   try {
		    	 String sql="select * from cvpr where ckeyword=? " ;
		    	 Object [] params= {key};
		    	 rs=DBUtil.executeQuery(sql, params);
		         while(rs.next()) {
		        	 int Id=rs.getInt("id");
		        	 String cname=rs.getString("cname");
		        	 String chref=rs.getString("chref");
		        	 String cabstract=rs.getString("cabstract");
		        	 cvf=new Cvf(Id,cname,chref,cabstract,key);
		        	 cvfs.add(cvf);
		         }
		     }catch(SQLException e) {
		    	 e.printStackTrace();
		     }catch(Exception e) {
		    	 e.printStackTrace();
		     }finally {
		    	 try {
	     			//先开的后关，后开的先关
	     		if(rs!=null)rs.close();
	     		if(DBUtil.pstmt!=null)DBUtil.pstmt.close();
	     		if(DBUtil.connection !=null)DBUtil.connection.close();
	     		}catch(SQLException e) {
	     			e.printStackTrace();
	     		}finally {
	     			
	     		}
		    	
		     }
	  	   return cvfs;
	     }
}

　　ListServlel.java

package servlet;

import java.io.IOException;
import java.util.List;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import dao.dao;
import entity.Cvf;


public class ListServlet extends HttpServlet {

	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		/**
		 * 这里是设置编码集，以避免出现乱码问题
		 */
		request.setCharacterEncoding("utf-8");
		String key= request.getParameter("keyword");
		response.setCharacterEncoding("utf-8");
		response.setContentType("text/html;charset=utf-8");
	    dao dao=new dao();
		
		List<Cvf> cvfs=dao.Query(key);
		System.out.println(cvfs);
		request.setAttribute("cvfs",cvfs);
		
		request.getRequestDispatcher("list.jsp").forward(request, response);
	}


	protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		// TODO Auto-generated method stub
		doGet(request, response);
	}

}

　　QueryServlet.java

package servlet;

import java.io.IOException;
import java.util.List;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import dao.dao;
import entity.Cvf;
import utils.Jsouputil;


public class QueryServlet extends HttpServlet {

	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		
		/**
		 * 这里是设置编码集，以避免出现乱码问题
		 */
		request.setCharacterEncoding("utf-8");
		response.setCharacterEncoding("utf-8");
		response.setContentType("text/html;charset=utf-8");
		/**
		 * 这是爬取数据
		 */
//		Jsouputil jsouptil=new Jsouputil();
//		try {
//			Jsouputil.testSelector();
//		} catch (Exception e) {
//			// TODO Auto-generated catch block
//		e.printStackTrace();
//		}

		dao dao=new dao();
		
		List<Cvf> cvfs=dao.Query();
		System.out.println(cvfs);
		request.setAttribute("cvfs",cvfs);
		
		request.getRequestDispatcher("show.jsp").forward(request, response);
	}


	protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		// TODO Auto-generated method stub
		doGet(request, response);
	}

}

　　DBUtil.java

package utils;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;


public class DBUtil {
	//数据库URL和账号密码
			public static String URL="jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=GB18030&useSSL=false&serverTimezone=GMT&allowPublicKeyRetrieval=true";//数据源  ！！！！注意若出现加载或者连接数据库失败一般是这里出现问题
			
			private static final String UNAME="root";
			private static final String UPWD="1234";
			public static PreparedStatement  pstmt=null;
			public static ResultSet  rs = null;
			public static Connection  connection=null;
			
			
     //增删改
	 public static boolean executeUpdate(String sql,Object [] params) {
		 boolean flag = false;
     	try {
     		
     	//a.导入驱动，加载具体的驱动类
     	Class.forName("com.mysql.cj.jdbc.Driver");
     	//b.与数据库建立连接
     	connection = DriverManager.getConnection(URL,UNAME,UPWD);
     	
     	pstmt = connection.prepareStatement(sql);
     	for(int i=0;i<params.length;i++) {
     		pstmt.setObject(i+1, params[i]);
     	}
     	int count=pstmt.executeUpdate();//返回值表示，增删改几条数据
     	//处理结果
     	if(count>0)
     	{
     		System.out.println("操作成功！！！");
     		flag=true;
     	}
     	
     	}catch(ClassNotFoundException e) {
     		e.printStackTrace();
     	}catch(SQLException e) {
     		e.printStackTrace();
     	}catch(Exception e){
     		e.printStackTrace();
     	}finally {
     		try {
     			//先开的后关，后开的先关
     		if(pstmt!=null)pstmt.close();
     		if(connection !=null)connection.close();
     		}catch(SQLException e) {
     			e.printStackTrace();
     		}finally {
     			
     		}
     	}
		return flag;
	 }
	//查
	 public static ResultSet executeQuery(String sql,Object [] params) {
		 
		     	try {
		     		
		     	//a.导入驱动，加载具体的驱动类
		     	Class.forName("com.mysql.cj.jdbc.Driver");
		     	//b.与数据库建立连接
		     	connection = DriverManager.getConnection(URL,UNAME,UPWD);
		     	
		     	pstmt = connection.prepareStatement(sql);
		     	if(params!=null) {
		     	for(int i=0;i<params.length;i++) {
		     		pstmt.setObject(i+1, params[i]);
		     	}
		     	}
		     	 rs = pstmt.executeQuery();
		     	 return rs;
		     	}catch(ClassNotFoundException e) {
		     		e.printStackTrace();
		     		return null;
		     	}catch(SQLException e) {
		     		e.printStackTrace();
		     		return null;
		     	}catch(Exception e){
		     		e.printStackTrace();
		     		return null;
		     	}
		     
		}
}

　　HttpclientPool.java

package utils;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import com.alibaba.fastjson.JSONObject;

public class HttpClientPool {
/**
 * 这是httpClient连接池
 * @throws Exception 
 */
	public static void HttpClientPool() {
		//创建连接池管理器
		PoolingHttpClientConnectionManager cm =new  PoolingHttpClientConnectionManager();
		
		
		//设置最大连接数
		cm.setMaxTotal(100);
		//设置每个主机的最大连接数
		cm.setDefaultMaxPerRoute(10);
		//使用连接池管理器发起请求
//		doGet(cm);
//		doPost(cm);
	}

public static String doPost(PoolingHttpClientConnectionManager cm) throws Exception {
	//从连接池中获取httpClient对象
		CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();

      	//2、输入网址,发起请求，创建httpPost对象
    	  HttpPost httpPost= new HttpPost("http://openaccess.thecvf.com/CVPR2019.py#");
    	  System.out.println("发起请求的信息："+httpPost);
    	  
    	  //Post使用，声明List集合，封装表单中的参数
    	  List<NameValuePair> params= new ArrayList<NameValuePair>();
    	  params.add(new BasicNameValuePair("",""));
    	  
    	  //创建表单的Entity对象,第一个参数是封装好的参数，第二个是编码
    	  UrlEncodedFormEntity formEntity= new UrlEncodedFormEntity(params,"utf8");
    	  
    	  //设置表单的Entity对象到Post请求中
    	  httpPost.setEntity(formEntity);
    	  
    	  
    	//配置请求信息
    	  RequestConfig config = RequestConfig.custom().setConnectTimeout(10000)//设置创建连接的最长时间，单位为毫秒
    	  .setConnectionRequestTimeout(50000)//设置获取连接的最长时间，单位为毫秒
    	  .setSocketTimeout(1000*1000)//设置传输数据的最长时间，单位为毫秒
    	  .build();
    	  //给请求设置请求信息
    	  httpPost.setConfig(config);
    	  
    	  CloseableHttpResponse response=null;
    	  String content=null;
    	  try {
    	  //3、按回车，发起请求，返回响应，使用httpClient对象发起请求
    	   response = httpClient.execute(httpPost);
    	  //解析响应，获取数据
    	  //判断状态码是否为两百
    	  if(response.getStatusLine().getStatusCode()==200) {
    		  HttpEntity httpEntity = response.getEntity();
    		  if(httpEntity!=null) {
    	          content = EntityUtils.toString(httpEntity, "utf8");
    			  System.out.println(content.length());
//    			  System.out.println(content);
    			  }
    	  }else {
    		  System.out.println("请求失败"+response);
    	  }
    	  }catch(Exception e) {
    		  e.printStackTrace();
    	  }finally {
    
    		  try {
    			//关闭response
    			  if(response!=null) {
    				  //关闭response 
    				  response.close();
    			  }
				//不关闭httpClient
				//httpClient.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
    	  }
		  return content;    	  	
}

public static String doGet(PoolingHttpClientConnectionManager cm) throws Exception {
	//从连接池中获取httpClient对象
	CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();

	//创建URIBuilder
	  URIBuilder uribuilder= new URIBuilder("http://openaccess.thecvf.com/CVPR2019.py#");
	  //设置参数：参数名+参数值,可设置多个
	  
	  //2、输入网址,发起请求，创建httpGet对象
	  HttpGet httpGet= new HttpGet(uribuilder.build());
	  System.out.println("发起请求的信息："+httpGet);
	  
	  //配置请求信息
	  RequestConfig config = RequestConfig.custom().setConnectTimeout(10000*10000)//设置创建连接的最长时间，单位为毫秒
	  .setConnectionRequestTimeout(10000*10000)//设置获取连接的最长时间，单位为毫秒
	  .setSocketTimeout(100000*1000000)//设置传输数据的最长时间，单位为毫秒
	  .build();
	  //给请求设置请求信息
	  httpGet.setConfig(config);
	  
	  
	  
	  
	  
	  CloseableHttpResponse response=null;
	  String content=null;
	  try {
	  //3、按回车，发起请求，返回响应，使用httpClient对象发起请求
	   response = httpClient.execute(httpGet);
	  //解析响应，获取数据
	  //判断状态码是否为两百
	  if(response.getStatusLine().getStatusCode()==200) {
		  HttpEntity httpEntity = response.getEntity();
		  if(httpEntity!=null) {
          content = EntityUtils.toString(httpEntity, "utf8");
//		  System.out.println(content.length());
//		  System.out.println(content);
		  }
	  }
	  }catch(Exception e) {
		  e.printStackTrace();
	  }finally {

		  try {
			  if(response!=null) {
				  //关闭response 
				  response.close();
			  }
			//不能关闭httpClient
			//httpClient.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	  }
	  return content; 
}
}

　　Jsouputil.java

package utils;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Set;
import java.util.HashMap;  
import java.util.Iterator;  
import java.util.Map;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import dao.dao;
import entity.Cvf;



/**
 * 这是使用Jsoup解析
 */
public class Jsouputil {

	/**
	 * 使用Selector选择器获取元素
	 */
	public static void testSelector()throws Exception{
		//获取Document对象
		HttpClientPool httpClientPool =new HttpClientPool();
		//创建连接池管理器
		PoolingHttpClientConnectionManager cm =new  PoolingHttpClientConnectionManager();
		//获取网页HTML字符串
		String content=httpClientPool.doGet(cm);
						
		//解析字符串
		Document doc = Jsoup.parse(content);
//		System.out.println(doc.toString());
	
		//[attr=value],利用属性获取
		Elements elements = doc.select("div[id=content]").select("dl").select("dt[class=ptitle]");
		System.out.println(elements.toString());
		
		Cvf cvf=new Cvf();
		dao dao=new dao();
		if(elements!=null) 
		{
		for(Element ele:elements)
		{
			String href="http://openaccess.thecvf.com/";
			String cname=ele.select("a").text();
			System.out.println(cname);
			String href2=ele.select("a").attr("href");
			String chref=href.concat(href2);
			System.out.println(chref);
			String cabstract =null;
			String ckeyword  =null;
				//获取title的内容
			CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
				//创建URIBuilder
				  URIBuilder uribuilder= new URIBuilder(chref);
				  HttpGet httpGet= new HttpGet(uribuilder.build());
				  RequestConfig config = RequestConfig.custom().setConnectTimeout(10000*10000)//设置创建连接的最长时间，单位为毫秒
						  .setConnectionRequestTimeout(10000*10000)//设置获取连接的最长时间，单位为毫秒
						  .setSocketTimeout(100000*1000000)//设置传输数据的最长时间，单位为毫秒
						  .build();
						  //给请求设置请求信息
						  httpGet.setConfig(config);
						  CloseableHttpResponse response=null;
						  response = httpClient.execute(httpGet);
		    	  //解析响应，获取数据
		    	  //判断状态码是否为两百
		    	  if(response.getStatusLine().getStatusCode()==200||response.getStatusLine().getStatusCode()==302) {
		    		  Document document = Jsoup.parse(new URL(chref), 100000);
				      cabstract = document.select("div[id=abstract]").text();
				      System.out.println("已获取摘要");
					String[] strs = strTostrArray(cname+cabstract);
					  
					 ckeyword=keyword(strs);
					 
		    	  }
		    	  else {
		    		  System.out.println(response.getStatusLine().getStatusCode());
		    		  cabstract =null;
		    		  ckeyword=null;
		    	  }
		    	  if(response!=null) {
					  //关闭response 
					  response.close();
				  }
		    	  
			cvf=new Cvf(cname,chref,cabstract,ckeyword);
			dao.add(cvf);

		}
		}
		
	}
	public static String[] strTostrArray(String str) {
		  /*
		   * 将非字母字符全部替换为空格字符" " 得到一个全小写的纯字母字符串包含有空格字符
		   */
		  str = str.toLowerCase();// 将字符串中的英文部分的字符全部变为小写
		  String regex = "[\\W]+";// 非字母的正则表达式 --\W：表示任意一个非单词字符
		  str = str.replaceAll(regex, " ");
		  String[] strs = str.split(" "); // 以空格作为分隔符获得字符串数组
		  return strs;
		 }
		 public static String keyword(String[] strs) {
		  /*
		   * 建立字符串(String)出现次数(Integer)的映射
		   */
		  HashMap<String, Integer> strhash = new HashMap<String, Integer>();
		  Integer in = null;// 用于存放put操作的返回值
		  for (String s : strs) {// 遍历数组 strs

				  in = strhash.put(s, 1);
				  if (in != null) {// 判断如果返回的不是null，则+1再放进去就是出现的次数
					  strhash.put(s, in + 1);
				  }

		  }
		  Set<java.util.Map.Entry<String, Integer>> entrySet = strhash.entrySet();
		  String maxStr = null;// 用于存放出现最多的单词
		  int maxValue = 0;// 用于存放出现最多的次数
		  for (java.util.Map.Entry<String, Integer> e : entrySet) {
		   String key = e.getKey();
		   Integer value = e.getValue();
		   if(key.equals("a")||key.equals("the")||key.equals("to")||key.equals("and")||key.equals("in")||key.equals("of")||key.equals("our")||key.equals("your")||key.equals("we")||key.equals("is")||key.equals("on")||key.equals("for")||key.equals("that")||key.equals("an")||key.equals("are")) {
			   value=0;
		   }
		   if (value > maxValue) {
		    maxValue = value;// 这里有自动拆装箱
		    maxStr = key;
		   }
		  }
		  System.out.println("出现最多的单词是：" + maxStr + "出现了" + maxValue + "次");
		  return maxStr;
		 }
}