HTMLPARSER JAVA CODE
1 /*** 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18 package org.apache.nutch.parse.html; 19 20 import java.util.ArrayList; 21 import java.util.Map; 22 import java.net.URL; 23 import java.net.MalformedURLException; 24 import java.nio.charset.Charset; 25 import java.io.*; 26 import java.util.regex.*; 27 28 import org.cyberneko.html.parsers.*; 29 import org.xml.sax.InputSource; 30 import org.xml.sax.SAXException; 31 import org.w3c.dom.*; 32 import org.apache.html.dom.*; 33 34 import org.apache.commons.logging.Log; 35 import org.apache.commons.logging.LogFactory; 36 37 import org.apache.nutch.metadata.Metadata; 38 import org.apache.nutch.metadata.Nutch; 39 import org.apache.nutch.protocol.Content; 40 import org.apache.hadoop.conf.*; 41 import org.apache.nutch.parse.*; 42 import org.apache.nutch.util.*; 43 44 public class HtmlParser implements Parser { 45 public static final Log LOG = LogFactory.getLog("org.apache.nutch.parse.html"); 46 47 // I used 1000 bytes at first, but found that some documents have 48 // meta tag well past the first 1000 bytes. 49 // (e.g. http://cn.promo.yahoo.com/customcare/music.html) 50 private static final int CHUNK_SIZE = 2000; 51 private static Pattern metaPattern = 52 Pattern.compile("<meta\\s+([^>]*http-equiv=\"?content-type\"?[^>]*)>", 53 Pattern.CASE_INSENSITIVE); 54 private static Pattern charsetPattern = 55 Pattern.compile("charset=\\s*([a-z][_\\-0-9a-z]*)", 56 Pattern.CASE_INSENSITIVE); 57 58 private String parserImpl; 59 60 /*** 61 * Given a <code>byte[]</code> representing an html file of an 62 * <em>unknown</em> encoding, read out 'charset' parameter in the meta tag 63 * from the first <code>CHUNK_SIZE</code> bytes. 64 * If there's no meta tag for Content-Type or no charset is specified, 65 * <code>null</code> is returned. <br /> 66 * FIXME: non-byte oriented character encodings (UTF-16, UTF-32) 67 * can't be handled with this. 68 * We need to do something similar to what's done by mozilla 69 * (http://lxr.mozilla.org/seamonkey/source/parser/htmlparser/src/nsParser.cpp#1993). 70 * See also http://www.w3.org/TR/REC-xml/#sec-guessing 71 * <br /> 72 * 73 * @param content <code>byte[]</code> representation of an html file 74 */ 75 76 private static String sniffCharacterEncoding(byte[] content) { 77 int length = content.length < CHUNK_SIZE ? 78 content.length : CHUNK_SIZE; 79 80 // We don't care about non-ASCII parts so that it's sufficient 81 // to just inflate each byte to a 16-bit value by padding. 82 // For instance, the sequence {0x41, 0x82, 0xb7} will be turned into 83 // {U+0041, U+0082, U+00B7}. 84 String str = ""; 85 try { 86 str = new String(content, 0, length, 87 Charset.forName("ASCII").toString()); 88 } catch (UnsupportedEncodingException e) { 89 // code should never come here, but just in case... 90 return null; 91 } 92 93 Matcher metaMatcher = metaPattern.matcher(str); 94 String encoding = null; 95 if (metaMatcher.find()) { 96 Matcher charsetMatcher = charsetPattern.matcher(metaMatcher.group(1)); 97 if (charsetMatcher.find()) 98 encoding = new String(charsetMatcher.group(1)); 99 } 100 101 return encoding; 102 } 103 104 private String defaultCharEncoding; 105 106 private Configuration conf; 107 108 private DOMContentUtils utils; 109 110 private HtmlParseFilters htmlParseFilters; 111 112 private String cachingPolicy; 113 114 public ParseResult getParse(Content content) { 115 HTMLMetaTags metaTags = new HTMLMetaTags(); 116 117 URL base; 118 try { 119 base = new URL(content.getBaseUrl()); 120 } catch (MalformedURLException e) { 121 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf()); 122 } 123 124 String text = ""; 125 String title = ""; 126 Outlink[] outlinks = new Outlink[0]; 127 Metadata metadata = new Metadata(); 128 129 // parse the content 130 DocumentFragment root; 131 try { 132 byte[] contentInOctets = content.getContent(); 133 InputSource input = new InputSource(new ByteArrayInputStream(contentInOctets)); 134 135 EncodingDetector detector = new EncodingDetector(conf); 136 detector.autoDetectClues(content, true); 137 detector.addClue(sniffCharacterEncoding(contentInOctets), "sniffed"); 138 String encoding = detector.guessEncoding(content, defaultCharEncoding); 139 140 metadata.set(Metadata.ORIGINAL_CHAR_ENCODING, encoding); 141 metadata.set(Metadata.CHAR_ENCODING_FOR_CONVERSION, encoding); 142 143 input.setEncoding(encoding); 144 if (LOG.isTraceEnabled()) { LOG.trace("Parsing..."); } 145 root = parse(input); 146 } catch (IOException e) { 147 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf()); 148 } catch (DOMException e) { 149 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf()); 150 } catch (SAXException e) { 151 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf()); 152 } catch (Exception e) { 153 e.printStackTrace(LogUtil.getWarnStream(LOG)); 154 return new ParseStatus(e).getEmptyParseResult(content.getUrl(), getConf()); 155 } 156 157 // get meta directives 158 HTMLMetaProcessor.getMetaTags(metaTags, root, base); 159 if (LOG.isTraceEnabled()) { 160 LOG.trace("Meta tags for " + base + ": " + metaTags.toString()); 161 } 162 // check meta directives 163 if (!metaTags.getNoIndex()) { // okay to index 164 StringBuffer sb = new StringBuffer(); 165 if (LOG.isTraceEnabled()) { LOG.trace("Getting text..."); } 166 utils.getText(sb, root); // extract text 167 text = sb.toString(); 168 sb.setLength(0); 169 if (LOG.isTraceEnabled()) { LOG.trace("Getting title..."); } 170 utils.getTitle(sb, root); // extract title 171 title = sb.toString().trim(); 172 } 173 174 if (!metaTags.getNoFollow()) { // okay to follow links 175 ArrayList<Outlink> l = new ArrayList<Outlink>(); // extract outlinks 176 URL baseTag = utils.getBase(root); 177 if (LOG.isTraceEnabled()) { LOG.trace("Getting links..."); } 178 utils.getOutlinks(baseTag!=null?baseTag:base, l, root); 179 outlinks = l.toArray(new Outlink[l.size()]); 180 if (LOG.isTraceEnabled()) { 181 LOG.trace("found "+outlinks.length+" outlinks in "+content.getUrl()); 182 } 183 } 184 185 ParseStatus status = new ParseStatus(ParseStatus.SUCCESS); 186 if (metaTags.getRefresh()) { 187 status.setMinorCode(ParseStatus.SUCCESS_REDIRECT); 188 status.setArgs(new String[] {metaTags.getRefreshHref().toString(), 189 Integer.toString(metaTags.getRefreshTime())}); 190 } 191 ParseData parseData = new ParseData(status, title, outlinks, 192 content.getMetadata(), metadata); 193 ParseResult parseResult = ParseResult.createParseResult(content.getUrl(), 194 new ParseImpl(text, parseData)); 195 196 // run filters on parse 197 ParseResult filteredParse = this.htmlParseFilters.filter(content, parseResult, 198 metaTags, root); 199 if (metaTags.getNoCache()) { // not okay to cache 200 for (Map.Entry<org.apache.hadoop.io.Text, Parse> entry : filteredParse) 201 entry.getValue().getData().getParseMeta().set(Nutch.CACHING_FORBIDDEN_KEY, 202 cachingPolicy); 203 } 204 return filteredParse; 205 } 206 207 private DocumentFragment parse(InputSource input) throws Exception { 208 if (parserImpl.equalsIgnoreCase("tagsoup")) 209 return parseTagSoup(input); 210 else return parseNeko(input); 211 } 212 213 private DocumentFragment parseTagSoup(InputSource input) throws Exception { 214 HTMLDocumentImpl doc = new HTMLDocumentImpl(); 215 DocumentFragment frag = doc.createDocumentFragment(); 216 DOMBuilder builder = new DOMBuilder(doc, frag); 217 org.ccil.cowan.tagsoup.Parser reader = new org.ccil.cowan.tagsoup.Parser(); 218 reader.setContentHandler(builder); 219 reader.setFeature(org.ccil.cowan.tagsoup.Parser.ignoreBogonsFeature, true); 220 reader.setFeature(org.ccil.cowan.tagsoup.Parser.bogonsEmptyFeature, false); 221 reader.setProperty("http://xml.org/sax/properties/lexical-handler", builder); 222 reader.parse(input); 223 return frag; 224 } 225 226 private DocumentFragment parseNeko(InputSource input) throws Exception { 227 DOMFragmentParser parser = new DOMFragmentParser(); 228 try { 229 parser.setFeature("http://cyberneko.org/html/features/augmentations", 230 true); 231 parser.setProperty("http://cyberneko.org/html/properties/default-encoding", 232 defaultCharEncoding); 233 parser.setFeature("http://cyberneko.org/html/features/scanner/ignore-specified-charset", 234 true); 235 parser.setFeature("http://cyberneko.org/html/features/balance-tags/ignore-outside-content", 236 false); 237 parser.setFeature("http://cyberneko.org/html/features/balance-tags/document-fragment", 238 true); 239 parser.setFeature("http://cyberneko.org/html/features/report-errors", 240 LOG.isTraceEnabled()); 241 } catch (SAXException e) {} 242 // convert Document to DocumentFragment 243 HTMLDocumentImpl doc = new HTMLDocumentImpl(); 244 doc.setErrorChecking(false); 245 DocumentFragment res = doc.createDocumentFragment(); 246 DocumentFragment frag = doc.createDocumentFragment(); 247 parser.parse(input, frag); 248 res.appendChild(frag); 249 250 try { 251 while(true) { 252 frag = doc.createDocumentFragment(); 253 parser.parse(input, frag); 254 if (!frag.hasChildNodes()) break; 255 if (LOG.isInfoEnabled()) { 256 LOG.info(" - new frag, " + frag.getChildNodes().getLength() + " nodes."); 257 } 258 res.appendChild(frag); 259 } 260 } catch (Exception x) { x.printStackTrace(LogUtil.getWarnStream(LOG));}; 261 return res; 262 } 263 264 public static void main(String[] args) throws Exception { 265 //LOG.setLevel(Level.FINE); 266 String name = args[0]; 267 String url = "file:"+name; 268 File file = new File(name); 269 byte[] bytes = new byte[(int)file.length()]; 270 DataInputStream in = new DataInputStream(new FileInputStream(file)); 271 in.readFully(bytes); 272 Configuration conf = NutchConfiguration.create(); 273 HtmlParser parser = new HtmlParser(); 274 parser.setConf(conf); 275 Parse parse = parser.getParse( 276 new Content(url, url, bytes, "text/html", new Metadata(), conf)).get(url); 277 System.out.println("data: "+parse.getData()); 278 279 System.out.println("text: "+parse.getText()); 280 281 } 282 283 public void setConf(Configuration conf) { 284 this.conf = conf; 285 this.htmlParseFilters = new HtmlParseFilters(getConf()); 286 this.parserImpl = getConf().get("parser.html.impl", "neko"); 287 this.defaultCharEncoding = getConf().get( 288 "parser.character.encoding.default", "windows-1252"); 289 this.utils = new DOMContentUtils(conf); 290 this.cachingPolicy = getConf().get("parser.caching.forbidden.policy", 291 Nutch.CACHING_FORBIDDEN_CONTENT); 292 } 293 294 public Configuration getConf() { 295 return this.conf; 296 } 297 }