JSOUP 暴力爬取实验
jsoup 是一款Java 的HTML解析器，可直接解析某个URL地址、HTML文本内容。它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据。
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.13.1</version>
</dependency>
下面用这个工具来暴力获取一个视频网站,各种视频的基本信息
建立获取信息的实体类,以便后期可以存入数据库
package cn.haidnor.movie;
import lombok.Data;
import java.util.List;
@Data
public class Movie {
// URL
private String url;
// 影片名
private String name;
// 年代
private String years;
// 国家
private String country;
// 时长
private String minute;
// 类型
private List<String> types;
// 导演
private List<String> director;
// 主演
private List<String> performers;
// 详细信息
private String details;
}
编写爬取工具
package cn.haidnor.movie;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
/**
* 全视频爬虫解析
* https://www.qsptv.net/
*
* @author haidnor
*/
public class QsptvReptile {
private static final int TIMEOUT = 8000;
private static int ip = 0;
/**
* 获取影片url资料
*
* @param url 视频连接
* @return Movie
*/
public Movie getMovie(String url,int id) throws Exception {
ip++;
Document doc = Jsoup.connect(url)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36")
.header("x-forwarded-for","1423" + ip + "")
.timeout(TIMEOUT).get();
Movie movie = new Movie();
// 设置 URL
movie.setUrl(url);
// 影片信息根元素
Element root = doc.getElementById("zanpian-score");
if (root == null) {
return movie;
}
// 影片名称
Element name = root.getElementsByTag("h1").get(0);
if (name != null) {
movie.setName(name.text());
}
// 年代
Element years = root.getElementsByClass("col-md-6 col-sm-6 col-xs-6 text hidden-xs").get(1);
if (years != null) {
years.getElementsByTag("span").remove();
movie.setYears(years.text());
}
// 国家
Element country = root.getElementsByClass("col-md-6 col-sm-6 col-xs-4 text hidden-xs").get(0).getElementsByTag("a").get(0);
if (country != null) {
movie.setCountry(country.text());
}
// 时长
Element minute = root.getElementsByClass("col-md-6 col-sm-6 col-xs-12 text").get(0);
if (minute != null) {
minute.getElementsByTag("span").remove();
movie.setMinute(minute.text());
}
// 类型
Elements types = root.getElementsByClass("col-md-12 text").get(0).getElementsByTag("a");
if (types != null) {
List<String> type = new ArrayList<String>();
for (Element element : types) {
type.add(element.text());
}
movie.setTypes(type);
}
// 主演
Elements performers = root.getElementsByClass("col-md-12 text").get(1).getElementsByTag("a");
if (performers != null) {
List<String> performer = new ArrayList<String>();
for (Element element : performers) {
performer.add(element.text());
}
movie.setPerformers(performer);
}
// 导演
Elements directors = root.getElementsByClass("col-md-6 col-sm-6 col-xs-12 text hidden-xs").get(1).getElementsByTag("a");
if (directors != null) {
List<String> director = new ArrayList<String>();
for (Element element : directors) {
director.add(element.text());
}
movie.setDirector(director);
}
// 影片详细信息
Element element = doc.getElementsByClass("details-content").last();
if (element != null) {
StringBuilder details = new StringBuilder(element.text());
int indexOf = details.lastIndexOf("全视频TV");
CharSequence charSequence = details.subSequence(0, indexOf);
movie.setDetails(charSequence.toString());
}
// 下载封面图片
Element picture = doc.getElementsByClass("video-pic").get(0);
StringBuilder style = new StringBuilder(picture.attr("style"));
String pictureUrl = style.substring(style.indexOf("(") + 1,style.indexOf(")"));
downloadPicture(pictureUrl,id);
return movie;
}
/**
* 下载图片
* @param pictureUrl
*/
static void downloadPicture(String pictureUrl,int id) throws Exception {
String filePath = "D:/picture";
File file = new File(filePath + "/" + id + ".jpg");
URL url = new URL(pictureUrl);
URLConnection connection = url.openConnection();
connection.setRequestProperty("User-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36");
connection.setRequestProperty("x-forwarded-for","143" + ip + "");
InputStream inputStream = connection.getInputStream();
DataInputStream dataInputStream = new DataInputStream(inputStream);
FileOutputStream fileOutputStream = new FileOutputStream(file);
ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[128];
int length;
while ((length = dataInputStream.read(buffer)) > 0) {
output.write(buffer, 0, length);
}
fileOutputStream.write(output.toByteArray());
output.close();
fileOutputStream.close();
dataInputStream.close();
}
public static void main(String[] args) throws Exception {
Movie movie = new QsptvReptile().getMovie("https://www.qsptv.net/show-2169.html",2169);
}
}
获取的元素需要自己去查看 HTML 来选择
开始爬取数据,这里开20个线程来获取数据
package cn.haidnor.movie;
public class Reptile implements Runnable {
// 资源最小值 1
private int index = 100;
// 资源最大值 83877
private int max = 9000;
static String urlPrefix = "https://www.qsptv.net/show-";
static String urlPostfix = ".html";
@Override
public void run() {
while (index <= max) {
String url = null;
try {
synchronized (this) {
url = urlPrefix + index + urlPostfix;
index++;
}
Movie movie = new QsptvReptile().getMovie(url, index);
System.out.println(movie);
} catch (Exception e) {
System.err.println("GET FAILED: " + url);
}
}
}
public static void main(String[] args) {
Reptile reptile = new Reptile();
for (int i = 0; i < 20; i++) {
new Thread(reptile).start();
}
}
}
可见获取数据的速度是非常快的。这个就是不做好网络安全的后果。服务器的压力会非常大。
