抓取HTML网页数据
（转）htmlparse filter使用
该类并不是一个通用的工具类，需要按自己的要求实现，这里只记录了Htmlparse.jar包的一些用法。仅此而已！
详细看这里：http://gundumw100.javaeye.com/blog/704311
import java.util.*; import org.htmlparser.Node; import org.htmlparser.NodeFilter; import org.htmlparser.Parser; import org.htmlparser.filters.AndFilter; import org.htmlparser.filters.HasAttributeFilter; import org.htmlparser.filters.NodeClassFilter; import org.htmlparser.filters.TagNameFilter; import org.htmlparser.tags.BodyTag; import org.htmlparser.tags.LinkTag; import org.htmlparser.util.NodeList; import org.htmlparser.util.ParserException; public class HtmlparseUtil { WebHttpClient util=new WebHttpClient(); public Map<String, String> linkGet(String url, String charset) { String content=util.getWebContentByGet(url,charset); Map<String, String> linkMap = new HashMap<String, String>(); try { //开始解析 Parser parser = Parser.createParser(content, charset); // 过滤出<a></a>标签 NodeFilter linkFilter = new NodeClassFilter(LinkTag.class); NodeList list = parser.extractAllNodesThatMatch(linkFilter); Node node = null; for (int i = 0; i < list.size(); i++) { node = list.elementAt(i); // 获得网页中的链接map(href,text) linkMap.put(((LinkTag) node).getLink(), this.processText(((LinkTag) node).getLinkText())); } } catch (ParserException e) { e.printStackTrace(); } return linkMap; } public String bodyGet(String url, String charset) { String content=util.getWebContentByGet(url,charset); String body = ""; try { Parser parser = Parser.createParser(content, charset); // 过滤<body></body>标签 NodeFilter bodyFilter = new NodeClassFilter(BodyTag.class); NodeList list = parser.extractAllNodesThatMatch(bodyFilter); Node node = null; for (int i = 0; i < list.size(); i++) { node = list.elementAt(i); // 获得网页内容 保存在content中 body = ((BodyTag) node).getBody(); } } catch (ParserException e) { e.printStackTrace(); } return body; } public Map<String,String> termGet(String url, String charset) { String content=util.getWebContentByGet(url,charset); Map<String, String> map = new HashMap<String, String>(); try { //开始解析 // 过滤出class为term的<span>元素 Parser parser = Parser.createParser(content, charset); AndFilter filter = new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","term")); Node node = null; NodeList nodeList = parser.parse(filter); for (int i = 0; i < nodeList.size(); i++) { node = nodeList.elementAt(i); map.put("term", node.toPlainTextString()); } // 过滤出class为start-time的<span>元素 Parser parser2 = Parser.createParser(content, charset); AndFilter filter2 = new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","start-time")); NodeList nodeList2 = parser2.parse(filter2); for (int i = 0; i < nodeList2.size(); i++) { node = nodeList2.elementAt(i); map.put("start-time", node.toPlainTextString()); } // 过滤出id为J_SingleEndTimeLabel的<span>元素 Parser parser3 = Parser.createParser(content, charset); AndFilter filter3 = new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("id","J_SingleEndTimeLabel")); NodeList nodeList3 = parser3.parse(filter3); for (int i = 0; i < nodeList3.size(); i++) { node = nodeList3.elementAt(i); map.put("end-time", node.toPlainTextString()); } // 过滤出class为box post的<div>元素 Parser parser4 = Parser.createParser(content, charset); AndFilter filter4 = new AndFilter(new TagNameFilter("div"),new HasAttributeFilter("class","box post")); NodeList nodeList4 = parser4.parse(filter4); for (int i = 0; i < nodeList4.size(); i++) { node = nodeList4.elementAt(i); String temp=node.toPlainTextString().trim(); temp=temp.substring(10,20).trim(); map.put("pre-term", temp); } // 过滤出class为J_AwardNumber的<span>元素 Parser parser5 = Parser.createParser(content, charset); // AndFilter filter5 = // new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","J_AwardNumber")); NodeList nodeList5 = parser5.parse(new HasAttributeFilter("class","J_AwardNumber")); StringBuffer buffer=new StringBuffer(); for (int i = 0; i < nodeList5.size(); i++) { node = nodeList5.elementAt(i); buffer.append(","+node.toPlainTextString()); } buffer.append("|"); // 过滤出class为blue J_AwardNumber的<span>元素 Parser parser6 = Parser.createParser(content, charset); // AndFilter filter6 = // new AndFilter(new TagNameFilter("span"),new HasAttributeFilter("class","blue J_AwardNumber")); NodeList nodeList6 = parser6.parse(new HasAttributeFilter("class","blue J_AwardNumber")); for (int i = 0; i < nodeList6.size(); i++) { node = nodeList6.elementAt(i); buffer.append(node.toPlainTextString()+","); } map.put("numbers", buffer.toString()); } catch (ParserException e) { // TODO Auto-generated catch block e.printStackTrace(); } return map; } private String processText(String content){ content=content.trim().replaceAll(" ", ""); // content=content.replaceAll("<p>", "\n"); // content=content.replaceAll("</TD>", ""); // content=content.replaceAll("</div>", ""); // content=content.replaceAll("</a>", ""); // content=content.replaceAll("<a href=.*>", ""); return content; } public static void main(String[] str) { String url="http://caipiao.taobao.com/lottery/order/lottery_dlt.htm?type=1"; HtmlparseUtil util=new HtmlparseUtil(); Map<String,String> map=util.termGet(url, "gb2312"); System.out.println("term="+map.get("term"));//<span class="term">第<em>10074</em>期</span> System.out.println("start-time="+map.get("start-time"));// System.out.println("end-time="+map.get("end-time"));// System.out.println("pre-term="+map.get("pre-term"));// System.out.println("numbers="+map.get("numbers"));// } }
