使用weka进行文本分类的例子

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import weka.classifiers.Classifier;
import weka.classifiers.bayes.NaiveBayes;
import weka.classifiers.trees.J48;
import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.Utils;
import weka.filters.Filter;
import weka.filters.unsupervised.attribute.StringToWordVector;

public class MessageClassify implements Serializable 
{
	private Instances instances = null;	
	private StringToWordVector filter = new StringToWordVector();
	private Classifier classifier = new NaiveBayes();
	
	/**
	 * 构造分类器，主要及时对数据格式，类标，类别数目等进行说明
	 */
	public MessageClassify() throws Exception
	{
		String nameOfDataset = "MessageClassification";		
		FastVector attributes = new FastVector(2);
		attributes.addElement(new Attribute("Message", (FastVector) null));
		FastVector classValues = new FastVector(2);//类标向量，共有两类
		classValues.addElement("alt.atheism");
		classValues.addElement("comp.graphics");
		attributes.addElement(new Attribute("Class", classValues));				
		instances = new Instances(nameOfDataset, attributes, 100);//可以把instance认为是行，attribute认为是列
		instances.setClassIndex(instances.numAttributes() - 1);//类表在instance中的那列
	}

	/**
	 * 添加数据到训练集中
	 */
	public void updateData(String message, String classValue) throws Exception
	{
		Instance instance = makeInstance(message, instances);
		instance.setClassValue(classValue);
		instances.add(instance);	
	}
	
	/**
	 * 文本分类要特别一点，因为在使用StringToWordVector对象计算文本中词项(attribute)权重的时候需要用到全局变量，比如DF，所以这里需要批量处理
	 * 在weka中要注意有些机器学习算法是批处理有些不是
	 */
	public void finishBatch() throws Exception
	{
		filter.setInputFormat(instances);
		Instances filteredData = Filter.useFilter(instances, filter);//这才真正产生符合weka算法输入格式的数据集
		classifier.buildClassifier(filteredData);//真正的训练分类器
	}
	
	/**
	 * 分类过程
	 */
	public void classifyMessage(String message) throws Exception
	{			
		filter.input(makeInstance(message, instances.stringFreeStructure()));
		Instance filteredInstance = filter.output();//必须使用原来的filter
		
		double predicted = classifier.classifyInstance(filteredInstance);//(int)predicted是类标索引
		System.out.println("Message classified as : "
				+ instances.classAttribute().value((int) predicted));
	}

	
	private Instance makeInstance(String text, Instances data)
	{		
		Instance instance = new Instance(2);		
		Attribute messageAtt = data.attribute("Message");		
		instance.setValue(messageAtt, messageAtt.addStringValue(text));		
		instance.setDataset(data);
		return instance;
	}

	public static String getStringFromFile(File file)
	{
		StringBuilder sb=new StringBuilder();
		try
		{
			BufferedReader br=new BufferedReader(new FileReader(file));
			String line;
			while(true)
			{				
				if((line=br.readLine())==null) break;
				sb.append(line.trim());
			}		
			br.close();
		} catch (Exception e){}
		return sb.toString();
	}
	
	
	static String modelname="weka.message";	
	public static void main(String[] options)
	{
		try
		{
			MessageClassify messageCl=null;
			if(new File(modelname).exists())
				messageCl=loadModel(modelname);
			else
			{
				messageCl=trainModel();
				try
				{
					ObjectOutputStream modelOutObjectFile = new ObjectOutputStream(new FileOutputStream(modelname));
					modelOutObjectFile.writeObject(messageCl);
					modelOutObjectFile.close();
				} catch (Exception e){}
			}
			String testPath="E:/datasets/20_newsgroups_two_class/test/alt.atheism/54262";
			messageCl.classifyMessage(getStringFromFile(new File(testPath)));		
		} 
		catch (Exception e){}
	}

	/**
	 * 训练分类器
	 */
	private static MessageClassify trainModel()
	{
		MessageClassify mc=null;
		try
		{
			mc = new MessageClassify();			
			String basePath="E:/datasets/20_newsgroups_two_class/train";
			File base=new File(basePath);
			for(File dir : base.listFiles())
			{
				for(File file : dir.listFiles())
				{
					String message=getStringFromFile(file);
					String classValue=dir.getName();
					mc.updateData(message, classValue);//添加一条训练样本，classvalue就是类标
				}				
			}
			mc.finishBatch();//训练过程
			
		} catch (Exception e){}
		return mc;
	}

	public static MessageClassify loadModel(String modelname)
	{
		MessageClassify mc=null;
		try
		{
			ObjectInputStream modelInObjectFile =new ObjectInputStream(new FileInputStream(modelname));
			mc = (MessageClassify) modelInObjectFile.readObject();
			modelInObjectFile.close();
		}
		catch (Exception e){}
		return mc;		
	}
}