Jsoup爬虫的简单使用

添加POM依赖

<dependency>
    <groupId>org.jsoup</groupId>
        <artifactId>jsoup</artifactId>
    <version>1.7.3</version>
</dependency>

JAVA代码示例

public static void main(String[] args) throws IOException{
        // 天眼查
        // String result1= HttpRequest.get("http://open.api.tianyancha.com/services/open/cb/ic/2.0?keyword=XXXX公司").header("Authorization", "").execute().body();;
        // System.err.println(result1);
        /*Document doc = Jsoup.connect("https://www.tianyancha.com/search?key=北京百度网讯科技有限公司").timeout(3000).get();
        System.err.println(doc.title());
        Elements newsHeadlines = doc.select(".cate_menu_lk");
        System.err.println(newsHeadlines.size());
        for (Element headline : newsHeadlines) {
          System.err.println( 
            headline.text());
        }
        */
        try {
            Document document = Jsoup.connect("https://www.so.com/s?ie=utf-8&fr=so.com&src=home_so.com&ssid=&q=java")
                    .timeout(5000)
                    .get();
             
            Elements elements = document.select(".res-title a");
  
            elements.forEach(element -> {
                System.out.println(element.text());
                System.err.println(element.attr("href"));
            });
            System.err.println("---------------------");
            for(int i=2;i<=10;i++){
                 Document documentt = Jsoup.connect("https://www.so.com/s?q=java&pn="+i+"&src=srp_paging&fr=so.com")
                         .timeout(5000)
                         .get();
                  
                 Elements eelements = documentt.select(".res-title a");
       
                 eelements.forEach(element -> {
                     System.out.println(element.text());
                     System.err.println(element.attr("href"));
                 });
                 System.err.println("---------------------");
            }
  
        } catch (IOException e) {
            e.printStackTrace();
        }
    }