java爬取携程酒店数据存到本地数据库
话不多说，直接上代码
1：配置依赖
</dependency>
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.8.3</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20160810</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>3.5.1</version>
</dependency>2：数据库表bean对象
public class Hotel {
/**
* 酒店id，酒店价格，酒店所在城市，酒店名称，酒店图片，酒店经纬度（地图），酒店地址,星级，评分
*/
private String hid;
private String price;
private String city;
private String name;
private String pic;
private String xpath;
private String ypath;
private String addr;
private String start;
private String score;
public String getHid() {
return hid;
}
public void setHid(String hid) {
this.hid = hid;
}
public String getPrice() {
return price;
}
public void setPrice(String price) {
this.price = price;
}
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getPic() {
return pic;
}
public void setPic(String pic) {
this.pic = pic;
}
public String getXpath() {
return xpath;
}
public void setXpath(String xpath) {
this.xpath = xpath;
}
public String getYpath() {
return ypath;
}
public void setYpath(String ypath) {
this.ypath = ypath;
}
public String getAddr() {
return addr;
}
public void setAddr(String addr) {
this.addr = addr;
}
public String getStart() {
return start;
}
public void setStart(String start) {
this.start = start;
}
public String getScore() {
return score;
}
public void setScore(String score) {
this.score = score;
}
@Override
public String toString() {
return "Hotel{" +
"hid='" + hid + '\'' +
", price='" + price + '\'' +
", city='" + city + '\'' +
", name='" + name + '\'' +
", pic='" + pic + '\'' +
", xpath='" + xpath + '\'' +
", ypath='" + ypath + '\'' +
", addr='" + addr + '\'' +
", start='" + start + '\'' +
", score='" + score + '\'' +
'}';
}
}3：数据库配置
driver=com.mysql.jdbc.Driver
url=jdbc:mysql://localhost:3306/数据库表名
username=***
password=***4：mybatis配置
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration
PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>
<properties resource="jdbc.properties"></properties>
<environments default="development">
<environment id="development">
<transactionManager type="JDBC"/>
<dataSource type="POOLED">
<property name="driver" value="${driver}"/>
<property name="url" value="${url}"/>
<property name="username" value="${username}"/>
<property name="password" value="${password}"/>
</dataSource>
</environment>
</environments>
<mappers>
<mapper resource="HotelMapper.xml"/>
</mappers>
</configuration>
5：dao接口，为插入数据做准备
public interface HotelMapper {
/**
* 插入数据
* @param hotel
* @return
*/
public int insert (Hotel hotel);
}6：做dao类的映射文件
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.yc.dao.HotelMapper">
<insert id="insert" parameterType="com.yc.model.Hotel">
insert into hotel(id,hid,name,price,city,xpath,ypath,pic,addr,start,score)
values (null,#{hid},#{name},#{price},#{city},#{xpath},#{ypath},#{pic},#{addr},#{start},#{score})
</insert>
</mapper>7：终极奥义，爬虫代码
public class Test02 {
public static void main(String[] args) throws IOException, InterruptedException {
Hotel hotel = new Hotel();
String resource = "mybatis-config.xml";
InputStream inputStream = null;
try {
//读取配置文件
inputStream = Resources.getResourceAsStream(resource);
} catch (IOException e) {
e.printStackTrace();
}
SqlSessionFactory sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);
//注册mybatis 工厂
//得到连接对象
SqlSession sqlSession = sqlSessionFactory.openSession(true);
//从mybatis中得到dao对象
HotelMapper mapper = sqlSession.getMapper(HotelMapper.class);
System.out.println(getHotelInfoByCity("guangzhou32"));
String str = "guangzhou32/p";
String str2;
int x=0;
for (int i = 1; i < 100; i+=2) {
str2 = str + i;
Thread.sleep(5000);
List<List<String>> lists = getHotelInfoByCity(str2);
for (List<String> list : lists) {
hotel.setHid(list.get(0));
hotel.setCity(list.get(1));
hotel.setPrice(list.get(2));
hotel.setName(list.get(3));
hotel.setXpath(list.get(4));
hotel.setYpath(list.get(5));
hotel.setPic(list.get(6));
hotel.setAddr(list.get(7));
hotel.setScore(list.get(8));
if (list.get(9)=="" ||list.get(9).isEmpty()){
list.set(9,"0");
}
hotel.setStart(list.get(9).charAt(list.get(9).length()-1)+"");
System.out.println(mapper.insert(hotel));
sqlSession.commit();
}
x+=25;
System.out.println("已爬取"+x+"条数据");
}
}
public static List<List<String>> getHotelInfoByCity(String city) throws IOException {
//首先输入要爬的网页
URL url = new URL("https://hotels.ctrip.com/hotel/" + city);
//建立连接
URLConnection urlConnection = url.openConnection();
HttpURLConnection connection = (HttpURLConnection) urlConnection;
// 建立一个读取流从连接中读取
BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(),
"utf-8"));
String current; // 爬虫爬下来的每一行数据
// 是否找到关键行1
boolean hadfindhtllst = false;
// 是否找到关键行2
boolean hadfindhotelPositionJSON = false;
// 内含hotelid,酒店剩余房间数
char[] htllist = new char[10005];
// 存放目标char[]->String
String roomnum = null;
// 内涵hotelid,酒店全称，酒店地址，酒店精确经纬度
char[] hotelPositionJSON = new char[50005];
// 存放目标char[]->String
String hotelinfo = null;
while ((current = in.readLine()) != null) {
// 如果还没找到关键行
if (hadfindhtllst == false) {
if (current.length() < 19) {
// 如果该行长度小于19 直接跳过
continue;
}
current.getChars(12, 19, htllist, 0);
if (charsequals(htllist, "htllist", 7)) {
// 找到了关建行
hadfindhtllst = true;
current.getChars(12, current.length(), htllist, 0);
roomnum = new String(htllist, 0, current.length() - 12);
}
}
// 如果还没找到关键行
if (hadfindhotelPositionJSON == false) {
if (current.length() < 25) {
// 如果改行长度小于25 直接跳过
continue;
}
current.getChars(8, 25, hotelPositionJSON, 0);
if (charsequals(hotelPositionJSON, "hotelPositionJSON", 17)) {
// 找到了关建行
hadfindhotelPositionJSON = true;
current.getChars(8, current.length(), hotelPositionJSON, 0);
hotelinfo = new String(hotelPositionJSON, 0, current.length() - 8);
}
}
}
// 经过爬虫后，有用的信息只有 String roomnum(内含酒店id，酒店剩余房间数)
// String hotelinfo(内含酒店id，酒店全名，酒店位置，酒店经纬度，酒店评分)
//System.err.println(roomnum+"........");
//System.err.println(hotelinfo+"--------");
if (roomnum == null || hotelinfo == null) {
return null;
}
List<List<String>> lists = new ArrayList<List<String>>();
// 添加酒店id，酒店城市，酒店余房数量
// cs为"出现的次数
for (int i = 0, cs = 0, now = 0; i < roomnum.length(); i++) {
if (roomnum.charAt(i) == '\"') {
cs++;
if (cs % 8 == 3) {
List<String> list = new ArrayList<String>();
String temp = "";
while (roomnum.charAt(++i) != '\"') {
temp += roomnum.charAt(i);
}
i--;
list.add(temp);
if (city.equals("chengdu28")) {
list.add("成都");
} else if (city.equals("guangzhou32")) {
list.add("广州");
} else if (city.equals("beijing1")) {
list.add("北京");
} else {
list.add("未知城市");
}
lists.add(list);
}
if (cs % 8 == 7) {
String temp = "";
while (roomnum.charAt(++i) != '\"') {
temp += roomnum.charAt(i);
}
i--;
lists.get(now++).add(temp);
}
}
}
// 添加酒店名，纬度lat，经度lon，酒店照片，酒店地址，酒店评分，酒店接待能力（星级）
for (int i = 0, cs = 0, now = 0; i < hotelinfo.length(); i++) {
if (hotelinfo.charAt(i) == '\"') {
cs++;
if (cs % 56 == 7 || cs % 56 == 11 || cs % 56 == 15
|| cs % 56 == 23 || cs % 56 == 27
|| cs % 56 == 31 || cs % 56 == 43) {
String temp = "";
while (hotelinfo.charAt(++i) != '\"') {
temp += hotelinfo.charAt(i);
}
i--;
lists.get(now).add(temp);
}
if (cs % 56 == 43) {
now++;
}
}
}
return lists;
}
// 查询char[]型 与 String型是否相等
public static boolean charsequals(char[] left, String right, int length) {
boolean result = true;
for (int i = length - 1; i >= 0; i--) {
result &= left[i] == right.charAt(i);
}
return result;
}
}
