scala爬取指定地点的所有列车班次
需求介绍:
爬取指定地点的所有全国相关的列车班次详情。将结果写进mysql。
步骤及所遇到的问题:
1.寻取全国站点静态信息 https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9002 一个静态的js文件
2.借助web工具,本人谷歌浏览器F12在12306相关页面找取所需要的接口
3.寻找规律,注意去重写进mysql
直接上代码,看注释:
object TrainSchedulesMain { def main(args: Array[String]): Unit = { // val dateStr = "2019-01-07" if (args == null || args.length < 2) { System.err.println("args is null or missing") System.exit(1) } val dateStrList = args(0).trim val station = args(1).trim assert(StringUtils.isNotBlank(dateStrList), "dateStrList is null or empty") assert(StringUtils.isNotBlank(station), "station is null or empty") // 打印参数 println(args.mkString(" ")) //获取全国所有的站点信息 val allStationsMap = analysisAllStations() /* * 1.深圳 * 2.深圳西 * 3.深圳东 * 4.深圳坪山 * 5.深圳北 * 6.福田 */ // val fromStations = collection.mutable.ListBuffer("深圳", "深圳西", "深圳东", "深圳坪山", "深圳北", "福田") var dateStr = ""; //出发站集合 val fromStations = collection.mutable.ListBuffer(station.trim) dateStrList.split(",").foreach(date => { dateStr = date fromStations.foreach(r => { var n = 0 //出发站 val fromStationRequest = allStationsMap.get(r).get allStationsMap.foreach(d => { val toStationRequest = d._2 //train code //根据出发站和到达站请求12306 excuteAnaly(fromStationRequest, toStationRequest) println(dateStr) println("进度:" + r) n += 1 println(n) }) }) }) def excuteAnaly(fromStationRequest: String, toStationRequest: String) = { val url1 = s"""https://kyfw.12306.cn/otn/leftTicket/queryZ?leftTicketDTO.train_date=${dateStr}&leftTicketDTO.from_station=${fromStationRequest}&leftTicketDTO.to_station=${toStationRequest}&purpose_codes=ADULT""" .stripMargin println("url1: " + url1) //获取所查询的 出发站-》到达站 的所有班次列车信息 val responseStr = HttpRequest.sendGet(url1) println("url1Res: " + responseStr) if (StringUtils.isNotBlank(responseStr) && JSONUtil.isJson(responseStr)) { val allStationModel = JSONUtil.toJavaBean(responseStr, new AllStationTimes().getClass) //表示初始站为深圳 目的地为 toStationRequest 是有值 if (allStationModel != null && allStationModel.getHttpstatus == 200 && allStationModel.getData.getResult != null && allStationModel.getData.getResult.size() > 0) { val resList = allStationModel.getData.getResult for (item <- 0 until resList.size()) { val str = resList.get(item) val indexNumStart = str.indexOf("预订") if (indexNumStart > -1) { val arrs = str.substring(indexNumStart).split("\\|") val trainNo = arrs(1) val trainCode = arrs(2) val startStation = arrs(3) val endStation = arrs(4) val fromStation = arrs(5) val toStation = arrs(6) // 并且目的地站必须是终点站才会当做一条 班次 写入msyql[否则是过站,导致重复写入mysql.并且站点还不全] if (toStation.trim.equals(endStation.trim)) { val url2 =s"""https://kyfw.12306.cn/otn/czxx/queryByTrainNo?train_no=${trainNo}&from_station_telecode=${fromStation}&to_station_telecode=${toStation}&depart_date=${dateStr}""" println("url2: " + url2) //查询上一步所有班次的详细各个站点顺序信息 val res = HttpRequest.sendGet(url2) println("url2Res: " + res) if (StringUtils.isNotBlank(res) && JSONUtil.isJson(res)) { val trainSchedulesModel = JSONUtil.toJavaBean(res, new TrainSchedules().getClass) if (trainSchedulesModel != null && trainSchedulesModel.getData.getData.size() > 0) { val data0 = trainSchedulesModel.getData.getData.get(0) if (!MysqlHandleUtil(MysqlConnect.trainDB).isHasValue( s"""select * | |from train_schedules |where train_code='${data0.getStation_train_code}' |and start_station_name='${data0.getStart_station_name}' |and end_station_name='${data0.getEnd_station_name}' |and start_time='${data0.getStart_time}' |and dates='$dateStr' |""" .stripMargin)) { val sql = s"""insert into |train_schedules(`train_code`,`start_station_name`,`end_station_name`,`start_time`,`arrive_time`,`dates`,`data`) |values('${data0.getStation_train_code}','${data0.getStart_station_name}','${data0.getEnd_station_name}','${data0.getStart_time}','${trainSchedulesModel.getData.getData.get(trainSchedulesModel.getData.getData.size() - 1).getArrive_time}','${dateStr}','${JSONUtil.toJsonString(trainSchedulesModel.getData.getData)}')""" .stripMargin MysqlHandleUtil(MysqlConnect.trainDB).insertData(sql) } } } } } } } } } } /** * 解析全国所有站点(中文名字,编码) * * @author XXXX * @date 17:08 * @param [] * @return scala.collection.mutable.HashMap<java.lang.String,java.lang.String> */ def analysisAllStations(): collection.mutable.HashMap[String, String] = { val hashMap = new mutable.HashMap[String, String]() val url = "https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9002" val response = HttpRequest.sendGet(url) assert(StringUtils.isNotBlank(response), "response is null or empty") val arrs = response.split("@") val len = arrs.length for (item <- 1 until len) { val ars = arrs(item).split("\\|") hashMap.put(ars(1), ars(2)) } hashMap } }
效果: