dom4j处理超大XML
dom4j提供了基于事件的模型来操作xml文档。利用该模型开发人员可以一部分、一部分的处理XML文档,而不需要将整个XML文档都加载到内存中。例如:假想你要处理一个非常大的XML文档,它可能是由数据库的某张数据表而来的。如下所示:
<ROWSET>
<ROW ID="1">
...
</ROW>
<ROW ID="2">
...
</ROW>
...
<ROW ID="N">
...
</ROW>
</ROWSET>
我们可以在某一时间只处理一个ROW节点,而不必立刻将文档的所有内容加载到内存中。dom4j提供一个基于事件的模型来实现它。我们可以注册一
个事件处理器来处理一个或多个路径表达式。事件处理器会在注册路径的开始和结束时被调用执行。当注册路径的开始标签找到时执行事件处理器的
onStart()方法,当注册路径的结束标签被找到时执行事件处理器的onEnd()方法。
onStart()和onEnd()方法传递一个ElementPath实例参数,这个实例既为根据注册路径遍历xml文档时的当前节点(Element)。如果想对遍历的当前节点进行操作,可以在onEnd()方法中对当前节点调用detach()方法保存改。
下面是示例代码:
- SAXReader reader = new SAXReader();
- reader.addHandler( "/ROWSET/ROW",
- new ElementHandler() {
- public void onStart(ElementPath path) {
- // do nothing here...
- }
- public void onEnd(ElementPath path) {
- // process a ROW element
- Element row = path.getCurrent();
- Element rowSet = row.getParent();
- Document document = row.getDocument();
- ...
- // prune the tree
- row.detach();
- }
- }
- );
- Document document = reader.read(url);
SAXReader reader = new SAXReader(); reader.addHandler( "/ROWSET/ROW", new ElementHandler() { public void onStart(ElementPath path) { // do nothing here... } public void onEnd(ElementPath path) { // process a ROW element Element row = path.getCurrent(); Element rowSet = row.getParent(); Document document = row.getDocument(); ... // prune the tree row.detach(); } } ); Document document = reader.read(url);
上面的办法解决了读的问题可是写的问题还没有解决。我命由我不由天(吹牛皮),畅游dom4j的doc文档找到如下几个方法
startDocument()
writeOpen();
writeClose();
endDocument()
动手一试,问题搞定,看来牛皮没白吹。下面是示例代码:
- /**
- * 数据写入xml文件
- * @param filePath 目标xml文件的存放路径
- * @return
- */
- public boolean writeXML(String filePath){
- XMLWriter out;
- try {
- /*
- * 创建XMLWriter对象,设置XML编码,解决中文问题。
- */
- OutputFormat outputFormat = OutputFormat.createPrettyPrint();
- outputFormat.setEncoding("GBK");
- out = new XMLWriter(new FileWriter(filePath),outputFormat);
- out.startDocument();
- Element rootElement = DocumentHelper.createElement("mans");
- out.writeOpen(rootElement);
- /*
- * 向mans节点写入子节点
- */
- for(int i=0 ; i<1000000 ; i++){
- Element man = createManElement(new Long(i), "shuhang"+i);//用于创建节点的方法
- out.write(man);
- System.out.println(" the loop index is : " + i);
- }
- out.writeClose(rootElement);
- out.endDocument();
- out.close();
- return true;
- } catch (IOException e) {
- e.printStackTrace();
- return false;
- } catch (SAXException e) {
- e.printStackTrace();
- return false;
- }
- }
/** * 数据写入xml文件 * @param filePath 目标xml文件的存放路径 * @return */ public boolean writeXML(String filePath){ XMLWriter out; try { /* * 创建XMLWriter对象,设置XML编码,解决中文问题。 */ OutputFormat outputFormat = OutputFormat.createPrettyPrint(); outputFormat.setEncoding("GBK"); out = new XMLWriter(new FileWriter(filePath),outputFormat); out.startDocument(); Element rootElement = DocumentHelper.createElement("mans"); out.writeOpen(rootElement); /* * 向mans节点写入子节点 */ for(int i=0 ; i<1000000 ; i++){ Element man = createManElement(new Long(i), "shuhang"+i);//用于创建节点的方法 out.write(man); System.out.println(" the loop index is : " + i); } out.writeClose(rootElement); out.endDocument(); out.close(); return true; } catch (IOException e) { e.printStackTrace(); return false; } catch (SAXException e) { e.printStackTrace(); return false; } }
新问题出现。对于读取数据的方法,在onEnd()方法中只是进行对Element的简单操作而已,若要对Element进行复杂的处理怎么办,
如将
Element节点的数据写入数据库,无法在onEnd()方法中加入过多的代码,因为无法通过编译。仔细想了一下dom4j的注册事件处理器应该用的是
模板方法,通过钩子将用户的操作加入到模板中去。何不自己写一个事件处理器来完成我们自己的定制操作,代码如下:
- public class ManElementHandler implements ElementHandler {
- public String mdbName;
-
- public ManElementHandler(){
- }
-
- public ManElementHandler(String mdbName){
- this.mdbName = mdbName;
- }
-
- public boolean saveMan(Element element, String mdbName){
- return true;
- }
-
- public void onEnd(ElementPath arg0) {
- Element row = arg0.getCurrent();
- Element rowSet = row.getParent();
- Document document = row.getDocument();
- Element root = document.getRootElement();
- Iterator it = root.elementIterator();
- while(it.hasNext()){
- Element element = (Element)it.next();
- System.out.println(" id : " + element.elementText("id") + " name : " + element.elementText("name"));
- saveMan(element, this.mdbName);
- }
- row.detach();
- }
- public void onStart(ElementPath path) {
- }
- }
public class ManElementHandler implements ElementHandler { public String mdbName; public ManElementHandler(){ } public ManElementHandler(String mdbName){ this.mdbName = mdbName; } public boolean saveMan(Element element, String mdbName){ return true; } public void onEnd(ElementPath arg0) { Element row = arg0.getCurrent(); Element rowSet = row.getParent(); Document document = row.getDocument(); Element root = document.getRootElement(); Iterator it = root.elementIterator(); while(it.hasNext()){ Element element = (Element)it.next(); System.out.println(" id : " + element.elementText("id") + " name : " + element.elementText("name")); saveMan(element, this.mdbName); } row.detach(); } public void onStart(ElementPath path) { } }
下面给出完整的实例代码。该实例首先创建一个xml文档,然后读取xml文档中的数据并将数据写入Access数据库中。测试时使用的文件大小为125M未发生内存溢出。
- //*****************************************************************************************************************
- package com;
- import java.sql.Connection;
- import java.sql.DriverManager;
- /**
- * 获取Access数据库的连接
- * @author 佛山无影脚
- * @version 1.0
- * Jul 7, 2008 4:35:49 PM
- */
- public class AccessMDBUtil {
- public static Connection connectMdb(String mdbName) {
- if(mdbName == null || mdbName.equals("")){
- return null;
- }
- try {
- Class.forName("sun.jdbc.odbc.JdbcOdbcDriver");
- String dburl ="jdbc:odbc:driver={Microsoft Access Driver (*.mdb)};DBQ="+mdbName;
- Connection conn=DriverManager.getConnection(dburl);
- return conn;
- } catch (Exception e) {
- e.printStackTrace();
- return null;
- }
- }
- }
- //*******************************************************************************************************************
- //*******************************************************************************************************************
- package com;
- import java.sql.Connection;
- import java.sql.ResultSet;
- import java.sql.SQLException;
- import java.sql.Statement;
- import java.util.Iterator;
- import org.dom4j.Document;
- import org.dom4j.Element;
- import org.dom4j.ElementHandler;
- import org.dom4j.ElementPath;
- /**
- * 定制的事件处理器
- * @author 佛山无影脚
- * @version 1.0
- * Jul 7, 2008 4:35:49 PM
- */
- public class ManElementHandler implements ElementHandler {
- public String mdbName;//数据库名称 含路径
-
- public ManElementHandler(){
- }
-
- public ManElementHandler(String mdbName){
- this.mdbName = mdbName;
- }
-
- /**
- *将Element节点数据保存到数据库
- *@param element 遍历XML文档时的当前节点
- *@param mdbName 数据库名称 含路径
- *@return
- */
- public boolean saveMan(Element element, String mdbName){
- System.out.println(" the method saveMan in ManElementHandler is execute!");
- Statement stmt = null;
- ResultSet rs = null;
- Connection conn = null;
- try {
- conn= AccessMDBUtil.connectMdb(mdbName);
- stmt=conn.createStatement();
- String sql = "insert into mans(id,name) values(" + element.elementText("id") + ",'" + element.elementText("name") + "')";
- stmt.executeUpdate(sql);
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- } finally {
- if(rs != null) {
- try {
- rs.close();
- } catch (SQLException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- if(stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- if(conn != null) {
- try {
- conn.close();
- } catch (SQLException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- }
- return true;
- }
-
- public void onEnd(ElementPath arg0) {
- Element row = arg0.getCurrent();
- Element rowSet = row.getParent();
- Document document = row.getDocument();
- Element root = document.getRootElement();
- Iterator it = root.elementIterator();
- while(it.hasNext()){
- Element element = (Element)it.next();
- System.out.println(" id : " + element.elementText("id") + " name : " + element.elementText("name"));
- saveMan(element, this.mdbName);
- }
- row.detach();
- }
- public void onStart(ElementPath path) {
- }
- }
- //*************************************************************************************************************************
- //*************************************************************************************************************************
- package com;
- import java.io.File;
- import java.io.FileWriter;
- import java.io.IOException;
- import org.dom4j.Document;
- import org.dom4j.DocumentException;
- import org.dom4j.DocumentHelper;
- import org.dom4j.Element;
- import org.dom4j.io.OutputFormat;
- import org.dom4j.io.SAXReader;
- import org.dom4j.io.XMLWriter;
- import org.xml.sax.SAXException;
- import org.xml.sax.XMLReader;
- /**
- * 测试
- * @author 佛山无影脚
- * @version 1.0
- * Jul 7, 2008 4:35:49 PM
- */
- public class ManTest {
- /**
- * 创建man节点
- * <man><id>1</id><name>佛山无影脚</name></man>
- * @param id
- * @param name
- * @return
- */
- public Element createManElement(Long id,String name){
- Element manElement = DocumentHelper.createElement("man");
- manElement.addElement("id").addText(id.toString());
- manElement.addElement("name").addText(name);
- return manElement;
- }
-
- /**
- * 数据写入xml文件
- * @param filePath 目标xml文件的存放路径
- * @return
- */
- public boolean writeXML(String filePath){
- XMLWriter out;
- try {
- /*
- * 创建XMLWriter对象,设置XML编码,解决中文问题。
- */
- OutputFormat outputFormat = OutputFormat.createPrettyPrint();
- outputFormat.setEncoding("GBK");
- out = new XMLWriter(new FileWriter(filePath),outputFormat);
- out.startDocument();
- Element rootElement = DocumentHelper.createElement("mans");
- out.writeOpen(rootElement);
- /*
- * 向mans节点写入子节点
- */
- for(int i=0 ; i<1000000 ; i++){
- Element man = this.createManElement(new Long(i), "shuhang"+i);
- out.write(man);
- System.out.println(" the loop index is : " + i);
- }
- out.writeClose(rootElement);
- out.endDocument();
- out.close();
- return true;
- } catch (IOException e) {
- e.printStackTrace();
- return false;
- } catch (SAXException e) {
- e.printStackTrace();
- return false;
- }
- }
-
- /**
- * 从xml文件中读取数据,并将数据写入Access数据
- * @param filePath xml文件的存放路径
- * @return
- */
- public boolean readXML(String filePath){
- ManElementHandler manElementHandler = new ManElementHandler("F:\\dom4j\\xmlTest.mdb");
- SAXReader reader = new SAXReader();
- reader.addHandler( "/mans/man", manElementHandler);
- Document document = null;
- try {
- File file = new File(filePath);
- document = reader.read(file);
- } catch (DocumentException e) {
- e.printStackTrace();
- return false;
- }
- return true;
- }
-
-
- public static void main(String[] args){
- XMLReader reader = null;
- long startTime = System.currentTimeMillis();
- ManTest mantest = new ManTest();
- mantest.writeXML("f:\\dom4j\\mans.xml");
- mantest.readXML("f:\\dom4j\\mans.xml");
- long endTime = System.currentTimeMillis();
- System.out.println(" is end! the millis is : " + (endTime - startTime));
- }
- }