java开发爬虫Deno
java开发爬虫Deno
身为一个程序员不会两三手爬虫怎么能在行业里立足啊,这是开发中自己写的一个java爬虫的Demo,供大家参考。
java爬虫的开发依赖于jsoup.jar
直接上代码
public static void main(String[] args) { HttpClient client = new DefaultHttpClient(); HttpGet httpGet = new HttpGet(); Map map = new HashMap(); try { //百度百科999感冒灵连接 String url = "http://baike.baidu.com/link?url=c95Y4QJym_d_wFKGmcibRTI_KIyj-X_tOjnlOGJS9qekgO1tmWaWnrn7QyAjqvZX8At7LbI1XIa69IBZWejiCXDVM0jkSBMnVZKKu4jeg-ef4TJkKCXEXWcGJ8DRGTuHxW4qWB3pNNU7Y0KdrbNvGK"; // get请求获取页面信息 String bb = doget(url); Document doc; //用jsoup接收页面信息 doc = Jsoup.parse(bb); // 选择所有div的class为para的标签 Elements news = doc.select("div[class=para]"); for (Element result : news) { //获取标签的内容并打印 String str=result.text(); System.out.println(str); } } catch (Exception e) { e.printStackTrace(); } } public static String doget(String path) { InputStream is = null; ByteArrayOutputStream baos = null; try { // 伪造referer 绕过防盗链设置 URL url = new URL(path.trim()); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); if (200 == conn.getResponseCode()) { byte[] buff = new byte[4096]; int count; ByteArrayOutputStream out = new ByteArrayOutputStream(4096); InputStream in = conn.getInputStream(); while ((count = in.read(buff)) != -1) { out.write(buff, 0, count); } conn.disconnect(); return out.toString("UTF-8"); } } catch (Exception e) { e.printStackTrace(); } finally { if (baos != null) { try { baos.close(); } catch (IOException e) { e.printStackTrace(); } } if (is != null) { try { is.close(); } catch (IOException e) { e.printStackTrace(); } } } return null; }
转载注明引用