crawler_基础之_java.net.HttpURLConnection 访问网络资源

java访问网络资源 由底层到封装  为  scoket==> java.net.HttpURLConnection==>HttpClient

这次阐述先 java.net.HttpURLConnection 的方式 ，好处是用导包 ，jdk原生自带的。

HtmlUtil 包含尝试重连（3次） ，编码识别，保存文件到磁盘

package com.cph.crawler.core.utils;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * 类说明：html有关的操作 <br>
 * 2012-9-22下午08:22:20创建<br>
 * 
 * @author cphmvp
 */
public final class HtmlUtil {
    public final static Log LOG = LogFactory.getLog(HtmlUtil.class);
    static String defaultEncoding = "utf-8";
    static HttpURLConnection httpURLConnection = null;
    static URL urlModel = null;
    // 链接超时时间
    static int connectTimeout = 100000;
    // 读取响应超时时间
    static int readTimeout = 100000;

    /**
     * 下载图片<br>
     * 
     * @param url
     *            图片的下载地址<br>
     * @param savePath
     *            保存路径<br>
     * @throws IOException
     */
    @SuppressWarnings("resource")
    public static void downloadAndSavePictureToDisk(String url, String savePath)
            throws IOException {
        urlModel = new URL(url);
        httpURLConnection = (HttpURLConnection) urlModel.openConnection();
        httpURLConnection.setConnectTimeout(connectTimeout);
        httpURLConnection.setReadTimeout(readTimeout);
        httpURLConnection.setDoOutput(true);
        InputStream is = httpURLConnection.getInputStream();
        BufferedReader rd = new BufferedReader(new InputStreamReader(is));
        FileOutputStream fw = null;
        File f = new File(savePath.substring(0, savePath.lastIndexOf("/")));

        if (!f.exists()) {
            f.mkdirs();
        }
        File eixtsFile = new File(savePath);
        if (eixtsFile.exists()) {
            return;
        }
        fw = new FileOutputStream(savePath, true);
        int num = -1;
        while ((num = is.read()) != (-1))// 是否读完所有数据
        {
            fw.write(num);// 将数据写往文件
        }
        rd.close();
        is.close();
        if (httpURLConnection != null) {
            httpURLConnection.disconnect();
        }

    }

    /**
     * 讲url后面的参数进行编码
     * 
     * @param url
     * @return
     * @throws UnsupportedEncodingException
     */
    private static String encodParamters(String url)
            throws UnsupportedEncodingException {
        String returnStr = new String(url);
        String regex = "=([^&]+)";
        Pattern p = Pattern.compile(regex);
        Matcher m = p.matcher(url);
        while (m.find()) {
            String replaceStr = m.group(1);
            returnStr = returnStr.replaceFirst(replaceStr,
                    URLEncoder.encode(replaceStr, "utf-8"));
        }
        return returnStr;
    }

    /**
     * 获取会话的JSESSIONID
     * 
     * @param url
     * @return
     */
    public static String getSession(String url) {
        String sessionId = "";
        try {
            urlModel = new URL(url);
            httpURLConnection = (HttpURLConnection) urlModel.openConnection();
            httpURLConnection.setConnectTimeout(connectTimeout);
            httpURLConnection.setReadTimeout(readTimeout);
            String cookieVal = null;
            String key = null;
            for (int i = 1; (key = httpURLConnection.getHeaderFieldKey(i)) != null; i++) {
                if (key.equalsIgnoreCase("set-cookie")) {
                    cookieVal = httpURLConnection.getHeaderField(i);
                    cookieVal = cookieVal.substring(0, cookieVal.indexOf(";"));
                    sessionId = sessionId + cookieVal + ";";
                }
            }

        } catch (MalformedURLException e) {
            LOG.error(e);
        } catch (IOException e) {
            LOG.error(e);
        }
        return sessionId;
    }

    /**
     * 下载页面</br>
     * 
     * @param page
     *            </br>
     * @return 页面源码
     * @throws IOException
     * @throws UnsupportedEncodingException
     */
    public static StringBuffer downloadHtml(String url,String encoding) {
        StringBuffer sb = new StringBuffer();
        BufferedReader in = null;
        int tryNum = 0;
        while (true) {
            try {
                if (tryNum > 1) {
                    String ecodingUrl = encodParamters(url);
                    urlModel = new URL(ecodingUrl);
                } else {
                    urlModel = new URL(url);
                }
                httpURLConnection = (HttpURLConnection) urlModel
                        .openConnection();
                httpURLConnection.setConnectTimeout(connectTimeout);
                httpURLConnection.setReadTimeout(readTimeout);
                httpURLConnection
                        .setRequestProperty("User-Agent",
                                "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)");
                String redirectUrl = httpURLConnection.getURL().toString();
                if (!redirectUrl.equals(url)) {
                    LOG.info(url + "重定向后为" + redirectUrl);
                }
                String charSetHeader = httpURLConnection
                        .getHeaderField("Content-Type");
                String charSet = null;
                if (charSetHeader != null) {
                    Pattern p = Pattern.compile("charset=[\"']?(.*?)['\"]");
                    Matcher m = p.matcher(charSetHeader);
                    if (m.find())
                        charSet = m.group(1).trim();
                    if (null == charSet) {
                        charSet = encoding;
                    }
                }

                charSet = (charSet == null ? encoding : charSet);
                in = new BufferedReader(new InputStreamReader(
                        httpURLConnection.getInputStream(), charSet));
                String inputLine;
                while ((inputLine = in.readLine()) != null) {
                    sb.append(inputLine + "\n");
                    inputLine = null;
                }
                if (in != null)
                    try {
                        in.close();
                    } catch (IOException e) {
                        LOG.error(e);
                    }
                if (httpURLConnection != null)
                    httpURLConnection.disconnect();
                break;
            } catch (Exception e) {
                if (tryNum++ == 3) {
                    LOG.error("download page error [ " + urlModel + " ] ");
                    return null;
                }
                LOG.warn(tryNum + "次下载失败", e);
            }
        }
        return sb;

    }
    /**
     * 下载页面</br>
     * 
     * @param page
     *            </br>
     * @return 页面源码
     * @throws IOException
     * @throws UnsupportedEncodingException
     */
    public static StringBuffer downloadHtml(String url) {
        StringBuffer sb = new StringBuffer();
        BufferedReader in = null;
        int tryNum = 0;
        while (true) {
            try {
                if (tryNum > 1) {
                    String ecodingUrl = encodParamters(url);
                    urlModel = new URL(ecodingUrl);
                } else {
                    urlModel = new URL(url);
                }
                httpURLConnection = (HttpURLConnection) urlModel
                        .openConnection();
                httpURLConnection.setConnectTimeout(connectTimeout);
                httpURLConnection.setReadTimeout(readTimeout);
                httpURLConnection
                        .setRequestProperty("User-Agent",
                                "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)");
                String redirectUrl = httpURLConnection.getURL().toString();
                if (!redirectUrl.equals(url)) {
                    LOG.info(url + "重定向后为" + redirectUrl);
                }
                String charSetHeader = httpURLConnection
                        .getHeaderField("Content-Type");
                String charSet = null;
                if (charSetHeader != null) {
                    Pattern p = Pattern.compile("charset=[\"']?(.*?)['\"]");
                    Matcher m = p.matcher(charSetHeader);
                    if (m.find())
                        charSet = m.group(1).trim();
                    if (null == charSet) {
                        charSet = defaultEncoding;
                    }
                }

                charSet = (charSet == null ? defaultEncoding : charSet);
                in = new BufferedReader(new InputStreamReader(
                        httpURLConnection.getInputStream(), charSet));
                String inputLine;
                while ((inputLine = in.readLine()) != null) {
                    sb.append(inputLine + "\n");
                    inputLine = null;
                }
                if (in != null)
                    try {
                        in.close();
                    } catch (IOException e) {
                        LOG.error(e);
                    }
                if (httpURLConnection != null)
                    httpURLConnection.disconnect();
                break;
            } catch (Exception e) {
                if (tryNum++ == 3) {
                    LOG.error("download page error [ " + urlModel + " ] ");
                    return null;
                }
                LOG.warn(tryNum + "次下载失败", e);
            }
        }
        return sb;

    }

}