获取网页的编码方式
package com.tl.spider.download; import com.tl.spider.utils.StaticValue; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; import java.net.URLConnection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * @ClassName CharsetDetectorUtil * @Description 给定一个url,获取其网页源代码的编码 * @Author Administrator * @Date 2019/5/19 13:57 * @Version 1.0 **/ public class CharsetDetectorUtil { /** * 获取网页源代码的编码方式 * @param url * @return * @throws IOException */ public static String getCharset(String url) throws Exception { String charSet = null; URL urlObject = new URL(url); URLConnection urlConnection = urlObject.openConnection(); Map<String, List<String>> map = urlConnection.getHeaderFields(); List<String> list = map.get("Content-Type"); if(list != null && !list.isEmpty()) { String line = list.get(0); String[] array = line.split(";"); for(String str : array) { String[] eleArray = str.split("="); if(eleArray.length == 2) { if(eleArray[0].equals("charset")) { charSet = eleArray[1].trim(); } } } } /** * 由于网页的编码方式的说明只在网页源代码的前几行,所以不需要获取所有的网页源代码 */ if(charSet == null) { // 启用meta获取网页的编码方式 BufferedReader bufferedReader = WebPageDownLoadUtil.getBR(url, StaticValue.ENCODING_DEFAULT); String tmp = null; while((tmp = bufferedReader.readLine()) != null) { tmp = tmp.toLowerCase(); String charset = getCharSetValue4Line(tmp); if(charset != null) { charSet = charset; break; } if(tmp.contains("</head>")) { break; } } if(bufferedReader != null) { bufferedReader.close(); } } return charSet; } /** * * @param line * @return */ public static String getCharSetValue4Line(String line) { String charsetValue = null; String regex = "charset=\"?(.+?)\"?\\s?/?>"; // 这个地方需要综合多个网页进行相应的修改 Pattern pattern = Pattern.compile(regex); Matcher matcher = pattern.matcher(line); if(matcher.find()) { charsetValue = matcher.group(1); } return charsetValue; } public static void main(String[] args) throws Exception { String url = "http://news.youth.cn/"; //String url = "https://www.baidu.com/"; //String url= "https://hao.360.com/?s0001"; String charSet = getCharset(url); System.out.println(charSet); } }
其中getBr函数为:
/** * 获取BufferedReader * @param url * @param charset * @return * @throws Exception */ public static BufferedReader getBR(String url, String charset) throws Exception { URL urlObject = new URL(url); InputStream inputStream = urlObject.openStream(); InputStreamReader inputStreamReader = new InputStreamReader(inputStream, charset); BufferedReader bufferedReader = new BufferedReader(inputStreamReader); return bufferedReader; }
积沙成塔