【2023.1.19】用Python写了一个爬虫获取58同城租房信息，并结合高德JS API在地图上进行显示，并进行路线规划
系统设计
- 用Python爬虫在58同城网上爬取房源信息，保存为本地CSV文件
- 申请高德开发者Key，用JS API示例编写Index.html，在web上设置工作地点，导入CSV房源信息文件，显示工作地点及其1小时到达范围内的房源信息，并进行工作地点到房源的路径规划
- 本地开启http server
Python爬虫crawler.py
- 因为要写入到同一csv文件，所以没用多线程，并且多线程爬取时存在封IP的问题
- 爬取58同城网上昆明翠湖地区房源的名字、网址和位置信息，如下图所示
import csv
import requests
from pyquery import PyQuery as pq
url = "https://km.58.com/cuihu/zufang/pn{page}/?key=%E7%A7%9F%E6%88%BF%E5%AD%90&final=1"
headers = {
'authority': 'km.58.com',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'zh-CN,zh;q=0.9',
'cookie': 'f=n; commontopbar_new_city_info=541%7C%E6%98%86%E6%98%8E%7Ckm; f=n; commontopbar_new_city_info=541%7C%E6%98%86%E6%98%8E%7Ckm; userid360_xml=CB19D4C9E81B140801B4C5065601FDA8; time_create=1676543927873; 58home=km; f=n; id58=pRXw3GPGehtsR5nJNS2dAg==; commontopbar_new_city_info=541%7C%E6%98%86%E6%98%8E%7Ckm; city=km; commontopbar_ipcity=km%7C%E6%98%86%E6%98%8E%7C0; 58tj_uuid=ff377cd1-7939-4285-9b85-398cb4563b6d; new_uv=1; utm_source=; spm=u-2few826hy988m2wxeg.2few8w84cwgt4eurg.kd_35028742829.cr_7724144035.cd_7549bb71213c6079; init_refer=https%253A%252F%252Fcn.bing.com%252F; als=0; wmda_uuid=e5032ea14914dcc52c01ae4f7add3a84; wmda_new_uuid=1; wmda_session_id_11187958619315=1673951776868-259b42ae-3b9e-a293; wmda_visited_projects=%3B11187958619315; new_session=0; __bid_n=185bf4d81ffcb588864207; FPTOKEN=SkjWhxEce3zRRV9ztPtC/RRkLN+EA/Bu23YM9sSDW6L+2NKTbKg0VK8/fFqrtafVdYN4arh/pwMuXPOGh6WC0uQB5wAIRFJGdgsc5aQRkuNgt3uk3j3M5HvpVOIcQc6kactWNyf9ZusWSlm+K1N+rkByI4E8dtFsHODPjEREqvmdsSvU9faobxszjPGIQj6JCom8I1bAxGj0WcNXt3Kh+9YzqFkEB6llgdd4++/qEbTT6uckUrekv2asxwOYbqOCeby+HTElXTBL4T0BSPw+GasPuMqwHb4ny4gr9HI/9Vev0vz/UjLwiw/6aqcs6AHAjEev747yFePHcZJca0V6CIgp3jCUThe1zlfHNIJo6Kq4CtSXmXucNSCuVOpvWhuckzx+HPwTGRjEO08MxvnXMWpvl8C/S3FG/CqwjlchcgE4i5PmFVHjGTs8XiUqTfLs|i0mv1G+Cd1oH+JvHe1ZzIhfbmxFrwfAhoOvHZRonvnU=|10|d30ba23a24900cd15a2ab321bc2069a0; xxzl_cid=d9630409896c4c58b437abc978df5d28; xxzl_deviceid=kMMqALshaA03PorzMMEEm68EJBeI4e0yX2c0IC%2FTOMuK%2BiJxgoH6RLBbjW5Fov3i',
'referer': 'https://km.58.com/cuihu/zufang/pn4/?key=%E7%A7%9F%E6%88%BF%E5%AD%90&final=1',
'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'same-origin',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
}
def get_page_num():
page_url = url.format(page=1)
content = requests.get(page_url, headers=headers)
doc = pq(content.text)
num = doc("#pager_wrap > div > a:nth-last-child(2) > span").text()
print ("页码数为：" + num)
return int(num)
def get_one_page(page):
page_url = url.format(page=page)
content = requests.get(page_url, headers=headers)
for h_name, h_loc, h_url in parse_page(content.text):
writer.writerow([h_name, h_loc, h_url])
print ("第" + str(page) + "页房源信息保存完毕")
def parse_page(html):
doc = pq(html)
items = doc("body > div.list-wrap > div.list-box > ul > li")
for item in items.children().items():
house_name = item.find("h2 > a").text()
if house_name != "":
house_url = item.find("h2 > a").attr('href').split("&dataSource")[0]
house_loc = item.find("p.infor > a:nth-child(2)").text()
yield house_name, house_loc, house_url
if __name__ == "__main__":
csv_file = open('renting.csv', 'w', encoding='utf_8_sig', newline='')
writer = csv.writer(csv_file, dialect='excel')
num = get_page_num()
for page in range(1, num+1):
get_one_page(page)
http服务器myweb.py
from http.server import HTTPServer, CGIHTTPRequestHandler
PORT = 8080
httpd = HTTPServer(("", PORT), CGIHTTPRequestHandler)
print ("在" + str(PORT) + "端口开启web服务器")
httpd.serve_forever()
高德JS API开发Web页面Index.html
- html设置工作地点名输入框，房源信息文件导入按钮
- 用JS API开发工作地点描绘功能，将房源信息加载到地图上，并设置工作地点通勤一小时的范围，以及规划工作地点到选中房源的路径，借助现成的demo以及jquery的事件方法来实现上述功能
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="initial-scale=1.0, user-scalable=no,
width=device-width">
<title>高德地图+58租房</title>
<link rel="stylesheet" href="http://cache.amap.com/lbs/static/main1119.css"/>
<link rel="stylesheet" href="http://cache.amap.com/lbs/static/jquery.range.css"/>
<script src="http://cache.amap.com/lbs/static/jquery-1.9.1.js"></script>
<script src="http://cache.amap.com/lbs/static/es5.min.js"></script>
<script src="http://webapi.amap.com/maps?v=1.3&key=7af41c9af70b3fd698f07b8f4508b2b9&plugin=AMap.ArrivalRange,AMap.Scale,AMap.Geocoder,AMap.Transfer,AMap.Autocomplete"></script>
<style>
/*面板控制样式*/
.control-panel {
position: absolute;
top: 30px;
left: 20px;
}
/*面板内容样式*/
.control-entry {
width: 290px;
background-color: rgba(119, 136, 153, 0.8);
font-family: fantasy, sans-serif;
text-align: left;
color: white;
overflow:hidden;
padding: 10px;
margin-bottom: 10px;
}
/*文字与右侧的距离*/
.control-input {
margin-left: 120px;
}
/*输入框宽度*/
.control-input input[type="text"] {
width: 160px;
}
/*文字样式*/
.control-panel label {
float: left;
width: 120px;
}
/*路线规划信息窗体样式*/
#transfer-panel {
position: absolute;
background-color: white;
max-height: 80%;
overflow-y: auto;
top: 150px;
left: 20px;
width: 250px;
}
</style>
</head>
<body>
<div id="container"></div>
<!--显示输入地址面板-->
<div class="control-panel">
<div class="control-entry">
<label>输入工作地点：</label>
<div class="control-input">
<input id="work-location" type="text">
</div>
</div>
<!--显示导入房源的面板-->
<div class="control-entry">
<label>导入房源文件：</label>
<div class="control-input">
<input type="file" name="file" onChange="importRentInfo(this)"/>
</div>
</div>
</div>
<div id="transfer-panel"></div>
<script>
//地图部分
var map = new AMap.Map("container", {
resizeEnable: true, //页面可调整大小
zoomEnable: true, //可缩放
center: [116.397428, 39.90923], //地图中心，这里使用的是北京的经纬度
zoom: 11 //缩放等级，数字越大离地球越近
});
//添加标尺
var scale = new AMap.Scale();
map.addControl(scale);
//经度，纬度，时间，通勤方式（默认是地铁＋公交）
var x, y, t, vehicle = "SUBWAY,BUS";
//工作地点，工作标记
var workAddress, workMarker;
//房源标记数组
var rentMarkerArray = [];
//多边形数组，存储到达范围的计算结果
var polygonArray = [];
//路线规划
var amapTransfer;
//到达范围对象
var arrivalRange = new AMap.ArrivalRange();
//信息窗体对象
var infoWindow = new AMap.InfoWindow({
offset: new AMap.Pixel(0, -30)
});
//地址自动补全对象
var auto = new AMap.Autocomplete({
//根据id指定输入内容
input: "work-location"
});
//添加事件监听，在选择完地址以后调用workLocationSelected
AMap.event.addListener(auto, "select", workLocationSelected);
//选择工作地点后触发的方法
function workLocationSelected(e) {
//更新工作地点，加载到达范围
workAddress = e.poi.name;
//调用加载1小时到达区域的方法
loadWorkLocation();
}
//加载工作地点标记
function loadWorkMarker(x, y, locationName) {
workMarker = new AMap.Marker({
map: map,
title: locationName,
icon: 'http://webapi.amap.com/theme/v1.3/markers/n/mark_r.png',
position: [x, y]
});
}
//清除已有的到达区域
function delWorkLocation() {
if (polygonArray) map.remove(polygonArray);
if (workMarker) map.remove(workMarker);
polygonArray = [];
}
//加载到达范围
function loadWorkRange(x, y, t, color, v) {
arrivalRange.search([x, y], t, function(status, result) {
if (result.bounds) {
for (var i = 0; i < result.bounds.length; i++) {
//多边形对象
var polygon = new AMap.Polygon({
map: map,
fillColor: color, //填充色
fillOpacity: "0.4", //透明度
strokeWeight: 1 //线宽
});
//到达范围的多边形路径
polygon.setPath(result.bounds[i]);
//增加多边形
polygonArray.push(polygon);
}
}
}, {
policy: v
});
}
//加载1小时到达区域
function loadWorkLocation() {
//清除已有的到达区域
delWorkLocation();
//创建地址坐标对象
var geocoder = new AMap.Geocoder({
city: "北京",
radius: 1000
});
//获取位置
geocoder.getLocation(workAddress, function(status, result) {
if (status === "complete" && result.info === 'OK') {
var geocode = result.geocodes[0]; //获取地址编码
x = geocode.location.getLng(); //经度
y = geocode.location.getLat(); //纬度
//加载工作地点标记
loadWorkMarker(x, y);
//加载工作地点1小时内到达的范围
loadWorkRange(x, y, 60, "#3f67a5", vehicle);
//地图移动到工作地点的位置
map.setZoomAndCenter(12, [x, y]);
}
})
}
//导入房源信息触发的方法
function importRentInfo(fileInfo) {
//获取房源文件名称
var file = fileInfo.files[0].name;
loadRentLocationByFile(file);
}
//清除现有的房源标记
function delRentLocation() {
if (rentMarkerArray) map.remove(rentMarkerArray);
rentMarkerArray = [];
}
//加载房源位置
function loadRentLocationByFile(fileName) {
//清除现有的房源标记
delRentLocation();
//所有的地点都记录在集合中
var rent_locations = new Set();
//获取文件中的房源信息
$.get(fileName, function(data) {
//分割信息
data = data.split("\n");
//遍历房源位置
data.forEach(function(item, index) {
rent_locations.add(item.split(",")[1]);
});
rent_locations.forEach(function(element, index) {
//加上房源标记
addMarkerByAddress(element);
});
});
}
//添加房源标记
function addMarkerByAddress(address) {
//地理编码对象
var geocoder = new AMap.Geocoder({
city: "北京",
radius: 1000
});
//获取位置
geocoder.getLocation(address, function(status, result) {
if (status === "complete" && result.info === 'OK') {
//获取地理编码
var geocode = result.geocodes[0];
//标记对象
rentMarker = new AMap.Marker({
map: map, //显示标记的地图
title: address, //鼠标移动至标记时所显示的文字
//标记图标地址
icon: 'http://webapi.amap.com/theme/v1.3/markers/n/mark_b.png',
//位置
position: [geocode.location.getLng(), geocode.location.getLat()]
});
rentMarkerArray.push(rentMarker);
//相关房源网络地址
rentMarker.content = "<div>房源：<a target = '_blank'href='http://bj.58.com/pinpaigongyu/?key=" + address + "'>" + address + "</a><div>"
//标记的事件处理
rentMarker.on('click', function(e) {
//设置信息窗体显示的内容
infoWindow.setContent(e.target.content);
infoWindow.open(map, e.target.getPosition());
//路线规划是否清除
if (amapTransfer) amapTransfer.clear();
//换乘对象
amapTransfer = new AMap.Transfer({
map: map,
policy: AMap.TransferPolicy.LEAST_TIME,
city: "北京市",
panel: 'transfer-panel'
});
//根据起、终点坐标查询换乘路线
amapTransfer.search([{
keyword: workAddress
}, {
keyword: address
}], function(status, result) {})
});
}
})
}
</script>
</body>
</html>
运行结果
- 爬取房源信息文件
- 主界面
- 输入工作地点
- 选择房源信息文件
- 显示房源位置标记，以及一小时通勤范围
- 规划工作地点到房源路线
