一个简单粗暴的爬虫 - 必应今日美图

1. 必应今日美图

当使用bing搜索时，每天都会出现一副美图。

搜索找到bing今日美图  http://bing.plmeizi.com/ (这里收集了一年多的今日美图) 收集者： http://leil.plmeizi.com/ 

目前共47页

url格式按 http://bing.plmeizi.com/?page=*

 

 点进去就是我们要的名称和名称

2. 开始编码

使用简单的Jsoup进行爬虫，很简单，很好理解。

HtmlUtil

package util;

import java.io.IOException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

public class HtmlUtil {
    // 根据url从网络获取网页文本
    public Document getHtmlTextByUrl(String url) {
        Document doc = null;
        try {
            // doc = Jsoup.connect(url).timeout(5000000).get();
            int i = (int) (Math.random() * 1000); // 做一个随机延时，防止网站屏蔽
            while (i != 0) {
                i--;
            }
            doc = Jsoup.connect(url).data("query", "Java").userAgent("Mozilla").cookie("auth", "token").timeout(300000)
                    .post();
        } catch (IOException e) {
            e.printStackTrace();
            try {
                doc = Jsoup.connect(url).timeout(5000000).get();
            } catch (IOException e1) {
                // TODO Auto-generated catch block
                e1.printStackTrace();
            }
        }
        return doc;
    }
}

GetPhoto

这个编码主要需要先分析html属性，按照属性取到我们需要的元素，元素值。

我是先拿到每个图的page页面url

然后到详情页面拿到图的url，图的名字截取。

然后将图保存到本地。

package bing;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import util.HtmlUtil;

/**
 *
 * @author loveincode
 * @data Sep 29, 2017 1:15:00 PM
 */
public class GetPhoto {

    public static void go(int startpage, int endpage) throws IOException {

        HtmlUtil htmlutil = new HtmlUtil();
        // 获取图片的绝对路径
        String url = "http://bing.plmeizi.com/?page=";
        for (int i = startpage; i <= endpage; i++) {
            String gourl = url + i + "";
            Document dochtml = htmlutil.getHtmlTextByUrl(gourl);
            Elements elements_a = dochtml.getElementsByClass("item");
            for (int x = 0; x < elements_a.size(); x++) {
                String pyotopage = elements_a.get(x).attr("href");
                Document dochtml_photo = htmlutil.getHtmlTextByUrl(pyotopage);
                Element elements_picurl = dochtml_photo.getElementById("picurl");
                String picurl = elements_picurl.attr("href");
                Element elements_searchlink = dochtml_photo.getElementById("searchlink");
                String name = elements_searchlink.getElementsByTag("span").get(0).html();
                name = name.split("\\(")[0];

                if (picurl.contains("jpg")) {
                    // 下载图片
                    URL url_pic = new URL(picurl);
                    DataInputStream dataInputStream = new DataInputStream(url_pic.openStream());
                    String imageName = name + ".jpg";
                    FileOutputStream fileOutputStream = new FileOutputStream(new File("bing_pic/" + imageName));
                    byte[] buffer = new byte[1024];
                    int length;
                    while ((length = dataInputStream.read(buffer)) > 0) {
                        fileOutputStream.write(buffer, 0, length);
                    }
                    dataInputStream.close();
                    fileOutputStream.close();
                }
            }
        }

    }

    public static void main(String[] args) throws IOException {
        System.out.println("test");
        go(1, 1);
    }

}

Mythread

package bing;

import java.io.IOException;

public class Mythread extends Thread {

    private int startpage;

    private int endpage;

    public Mythread(int startpage, int endpage) {
        this.startpage = startpage;
        this.endpage = endpage;
    }

    @SuppressWarnings("static-access")
    @Override
    public void run() {
        GetPhoto getPhoto = new GetPhoto();
        try {
            getPhoto.go(startpage, endpage);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
}RUN

RUN

采用多线程，开启多个线程同时爬取图片

package bing;

import java.io.IOException;

/**
 *
 * @author loveincode
 * @data Sep 29, 2017 1:55:57 PM
 */
public class RUN {

    public static void main(String[] args) throws IOException {

        long startTime = System.currentTimeMillis(); // 获取开始时间

        Mythread a1 = new Mythread(1, 5);
        Mythread a2 = new Mythread(6, 10);
        Mythread a3 = new Mythread(11, 15);
        Mythread a4 = new Mythread(16, 20);
        Mythread a5 = new Mythread(21, 25);
        Mythread a6 = new Mythread(26, 30);
        Mythread a7 = new Mythread(31, 35);
        Mythread a8 = new Mythread(36, 40);
        Mythread a9 = new Mythread(41, 45);
        Mythread a10 = new Mythread(46, 47);

        a1.start();
        a2.start();
        a3.start();
        a4.start();
        a5.start();
        a6.start();
        a7.start();
        a8.start();
        a9.start();
        a10.start();

        while (true) {
            if (a1.isAlive() == false && a2.isAlive() == false && a3.isAlive() == false && a4.isAlive() == false
                    && a5.isAlive() == false && a6.isAlive() == false && a7.isAlive() == false && a8.isAlive() == false
                    && a9.isAlive() == false && a10.isAlive() == false) {
                long endTime = System.currentTimeMillis(); // 获取结束时间
                System.out.println("程序运行时间： " + (endTime - startTime) / 1000.0 + "s");
                break;
            }
        }
    }

}

执行 RUN 

耗时76.962s 完成图片下载到本地。

成功

效果：

很高清吧