java类实现 指定网站信息采集

话不多出，直接写出实现过程。若是需要代理，就设置代理

//  // 设置代理上外网
//  System.getProperties().put("proxySet", "true");
//  System.getProperties().put("proxyHost", "172.31.170.14");
//  System.getProperties().put("proxyPort", "8080");

 

package com.chen.downMessage;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import com.chen.system.util.ChenFile;
import com.chen.system.util.ChenTools;

public class DownMessage
{
    public String saveRoot = "F:" + File.separator + "download" + File.separator;
    private String httpUrl0 = "http://www.yi-see.com/";

    // 详情页获取信息标签

    public DownMessage()
    {
    }

    public DownMessage(String saveRoot, String saveRoota)
    {
    this.saveRoot = saveRoot;
    this.httpUrl0 = httpUrl0;
    }

    /**
     * 获取url中的链接
     * 
     * @author chen_weixian
     * @param parentUrl
     * @param urlStartTag
     * @param urlEndTag
     * @return
     * @throws Exception
     */
    public List<String[]> downLoadMessage(String parentUrl, String areaStartTag, String areaEndTag, String hrefStartTag, String hrefEndTag, String saveFileTitleStartTag, String saveFileTitleEndTag) throws Exception
    {
    List<String[]> resultList = new ArrayList<String[]>();
    String htmlContent = this.getContentByUrl(parentUrl);
    // StringBuffer sb = new StringBuffer(); // 本地测试
    // ChenFile.readFile("F:/download/source.txt", sb, "gbk");
    // String htmlContent = sb.toString();

    String contentString = htmlContent.substring(htmlContent.indexOf(areaStartTag) + areaStartTag.length(), htmlContent.length());
    contentString = contentString.substring(0, contentString.indexOf(areaEndTag));

    String[] titleArray = contentString.split(saveFileTitleStartTag);
    if (titleArray.length > 0)
    {
        for (int i = 0; i < titleArray.length; i++)
        {
        try
        {
            if (titleArray[i].indexOf(hrefEndTag) <= 0 || titleArray[i].indexOf(saveFileTitleEndTag) <= 0)
            {
            continue;
            }
            String[] tempArray = new String[2];
            String url = titleArray[i].substring(titleArray[i].indexOf(hrefStartTag) + hrefStartTag.length(), titleArray[i].length());
            url = url.substring(0, url.indexOf(hrefEndTag));
            String title = ChenTools.clearHtml(titleArray[i]);
            // 获取标题，获取
            // String content = this.downStart(httpUrl0 + url);
            // this.saveFile(content, title);
            // System.out.println(i + "\t完成：" + title + "\t" + httpUrl0
            // + url);
            tempArray[0] = url;
            tempArray[1] = title;

            resultList.add(tempArray);
        }
        catch (Exception e)
        {
            // System.out.println(i + " 信息异常：" + e);
            continue;
        }
        }
    }

    return resultList;
    }

    /**
     * 读取信息内容
     * 
     * @author chen_weixian
     * @param urlString
     * @return
     * @throws MException
     */
    public String downStart(String urlString, String titleStartTag, String titleEndTag, String authorSt1artTag, String authorEndTag, String contentStartTag, String contentEndTag) throws Exception
    {
    String result = "";

//        // 设置代理上外网
//        System.getProperties().put("proxySet", "true");
//        System.getProperties().put("proxyHost", "172.31.170.14");
//        System.getProperties().put("proxyPort", "8080");
    String htmlContent = this.getContentByUrl(urlString);
    if (htmlContent.length() > 0)
    {
        StringBuffer resultString = new StringBuffer(100); // 结果
        String br = "    \r\n";
        // 标题
        if (!ChenTools.isEmpty(titleStartTag) && !ChenTools.isEmpty(titleEndTag))
        {
        String title = htmlContent.substring(htmlContent.indexOf(titleStartTag), htmlContent.length());
        title = title.substring(0, title.indexOf(titleEndTag));
        resultString.append(title);
        resultString.append(br);
        }
        // 作者
        if (!ChenTools.isEmpty(authorSt1artTag) && !ChenTools.isEmpty(authorEndTag))
        {
        String author = htmlContent.substring(htmlContent.indexOf(authorSt1artTag), htmlContent.length());
        author = author.substring(0, author.indexOf(authorEndTag));
        resultString.append(author);
        resultString.append(br);
        }
        // 内容
        if (!ChenTools.isEmpty(contentStartTag) && !ChenTools.isEmpty(contentEndTag))
        {
        String content = htmlContent.substring(htmlContent.indexOf(contentStartTag), htmlContent.length());
        content = content.substring(0, content.indexOf(contentEndTag));
        resultString.append(content);
        resultString.append(br);
        }

        result = resultString.toString().replaceAll("<br>", br);
    }
    // 去除html
    return ChenTools.clearHtml(result.toString());
    }

    /**
     * 读取url内容
     * 
     * @author chen_weixian
     * @param urlString
     * @return
     * @throws Exception
     */
    public String getContentByUrl(String urlString) throws Exception
    {
    StringBuffer htmlContent = new StringBuffer(); // 临时变量
    URL urlObj = new URL(urlString);
    HttpURLConnection httpcon = (HttpURLConnection) urlObj.openConnection();
    BufferedReader reader = new BufferedReader(new InputStreamReader(httpcon.getInputStream()));
    String line = "";
    while ((line = reader.readLine()) != null)
    {
        line = new String(line.getBytes(), "gbk");
        htmlContent.append(line);
    }
    reader.close();
    return htmlContent.toString();
    }

    /**
     * 保存文件
     * 
     * @author chen_weixian
     * @throws IOException
     */
    public void saveFile(String contentString, String title) throws IOException
    {
    String savepath = saveRoot + File.separator + title + ".txt";
    ChenFile.unExitCreate(savepath);
    ChenFile.WriteToFile(savepath, contentString, "gbk", true);
    }

    public void doMain(String parentUrl)
    {

    String titleStartTag1 = "<B>";
    String titleEndTag1 = "</B>";
    String authorSt1artTag1 = "</B><BR>";
    String authorEndTag1 = "</a><br>";
    String contentStartTag1 = "<TD CLASS=ART>";
    String contentEndTag1 = "</TD>";
    // 列表链接
    String areaStartTag2 = "<TABLE WIDTH=900px ALIGN=CENTER cellpadding=\"0\" cellspacing=\"0\" border=\"0\">";
    String areaEndTag2 = "</TABLE>";
    String hrefStartTag2 = "<a href='";
    String hrefEndTag2 = "' >";
    String saveFileTitleStartTag2 = "<BR>";
    String saveFileTitleEndTag2 = "</A>";
    // 获取一级链接页面
    String areaStartTag3 = "<TABLE WIDTH=900px ALIGN=CENTER cellpadding='0' cellspacing='0'>";
    String areaEndTag3 = "</TABLE>";
    String hrefStartTag3 = "<a href='";
    String hrefEndTag3 = "'";
    String saveFileTitleStartTag3 = "<TR>";
    String saveFileTitleEndTag3 = "</TR>";
    // 记录日志文件
    String logFile = "F:" + File.separator + "download" + File.separator + ChenTools.getCurrDatetime(2) + "下载日志.log";
    
    
    try
    {
        // 获取二级链接
        List<String[]> firstList = this.downLoadMessage(this.httpUrl0 + parentUrl, areaStartTag3, areaEndTag3, hrefStartTag3, hrefEndTag3, saveFileTitleStartTag3, saveFileTitleEndTag3);
        if (firstList != null && firstList.size() > 0)
        {
        System.out.println("firstList.size()=" + firstList.size());
        for (int i = 0; i < firstList.size(); i++)
        {
            String[] array = firstList.get(i);
            // 获取一级链接
            this.saveRoot = "F:" + File.separator + "download" + File.separator + array[1];
            // 三级链接
            List<String[]> lastList = this.downLoadMessage(this.httpUrl0 + array[0], areaStartTag2, areaEndTag2, hrefStartTag2, hrefEndTag2, saveFileTitleStartTag2, saveFileTitleEndTag2);
            if (lastList != null && lastList.size() > 0)
            {
            System.out.println("lastList.size()=" + lastList.size());
            for (int j = 0; j < lastList.size(); j++)
            {
                try
                {
                String[] array1 = lastList.get(j);
                String contentString = this.downStart(this.httpUrl0 + array1[0], titleStartTag1, titleEndTag1, authorSt1artTag1, authorEndTag1, contentStartTag1, contentEndTag1);
                this.saveFile(contentString, array1[1]);
                String message = i + "\t"+ array[1] + "\t" + j + "\t" + array1[1] + "\t" + array1[0];
                ChenFile.WriteToFile(logFile, message, "gbk", true);
                System.out.println(message);
                }
                catch (Exception e)
                {
                String message = i + "\t"+ "信息异常：\t" + array[1] + "\t" + j + "\n" + e;
                ChenFile.WriteToFile(logFile, message, "gbk", true);
                System.out.println(message);
                continue;
                }
            }
            }
        }
        }
    }
    catch (Exception e)
    {
        e.printStackTrace();
    }
    // 获取二级链接
    // 获取详情内容
    }

    /**
     * @author chen_weixian
     * @param args
     */
    public static void main(String[] args)
    {
    DownMessage downMessage = new DownMessage();
    String parentUrl = "artlist_3.html";
    downMessage.doMain(parentUrl);
    }

}