3.13号网上爬取疫情数据
package getDataTest; import java.io.IOException; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; import javax.naming.InitialContext; import javax.net.ssl.HttpsURLConnection; import org.apache.commons.dbutils.QueryRunner; import org.jsoup.Jsoup; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.mchange.v2.c3p0.DataSources; import utils.DataSourceUtils; public class GetYiQing { public static String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:49.0) Gecko/20100101 Firefox/49.0"; public static String HOST = "i.snssdk.com"; public static String REFERER = "https://i.snssdk.com/feoffline/hot_list/template/hot_list/forum_tab.html?activeWidget=1"; public static void main(String[] args) throws IOException, SQLException { String url = "https://i.snssdk.com/forum/home/v1/info/?activeWidget=1&forum_id=1656784762444839"; String resultBody; try { SSL.trustAllHttpsCertificates(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } HttpsURLConnection.setDefaultHostnameVerifier(SSL.hv); resultBody = Jsoup.connect(url). userAgent(USER_AGENT).header("Host", HOST).header("Referer", REFERER).execute().body(); JSONObject jsonObject = JSON.parseObject(resultBody); String ncovStringList = jsonObject.getJSONObject("forum").getJSONObject("extra").getString("ncov_string_list"); JSONObject ncovListObj = JSON.parseObject(ncovStringList); JSONArray todaydata = ncovListObj.getJSONArray("provinces"); QueryRunner queryRunner = new QueryRunner(DataSourceUtils.getDataSource()); String sql = "insert into info4 values(?,?,?,?,?,?,?,?)"; String confirmedNum,deathsNum,cityname,cityid,treatingNum,provinceid; String reprovinceid=null; int confirmedNumSum=0,deathsNumSum=0,treatingNumSum=0; for(int i=0;i<todaydata.size();i++) { JSONObject todayData1 = todaydata.getJSONObject(i); String updateDate = todayData1.getString("updateDate"); JSONArray city = todayData1.getJSONArray("cities"); for(int j=0;j<city.size();j++) { JSONObject cities = city.getJSONObject(j); confirmedNum= cities.getString("confirmedNum"); deathsNum = cities.getString("deathsNum"); cityname = cities.getString("name"); cityid = cities.getString("id"); treatingNum = cities.getString("treatingNum"); provinceid = cityid.substring(0,2); reprovinceid=provinceid; confirmedNumSum+=Integer.parseInt(confirmedNum); deathsNumSum+=Integer.parseInt(deathsNum); treatingNumSum+=Integer.parseInt(treatingNum); queryRunner.update(sql, updateDate,provinceid,cityname,confirmedNum,deathsNum,treatingNum,cityid,null); } queryRunner.update(sql,updateDate,reprovinceid,null,confirmedNumSum,deathsNumSum,treatingNumSum,null,null); confirmedNumSum=0; deathsNumSum=0; treatingNumSum=0; } } }
package getDataTest; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLSession; public class SSL { static HostnameVerifier hv = new HostnameVerifier() { public boolean verify(String urlHostName, SSLSession session) { //System.out.println("Warning: URL Host: " + urlHostName + " vs. " + session.getPeerHost()); return true; } }; static void trustAllHttpsCertificates() throws Exception { javax.net.ssl.TrustManager[] trustAllCerts = new javax.net.ssl.TrustManager[1]; javax.net.ssl.TrustManager tm = new miTM(); trustAllCerts[0] = tm; javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance("SSL"); sc.init(null, trustAllCerts, null); javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); } static class miTM implements javax.net.ssl.TrustManager, javax.net.ssl.X509TrustManager { public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } public boolean isServerTrusted(java.security.cert.X509Certificate[] certs) { return true; } public boolean isClientTrusted(java.security.cert.X509Certificate[] certs) { return true; } public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) throws java.security.cert.CertificateException { return; } public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) throws java.security.cert.CertificateException { return; } } }