TF-IDF

参考源：
http://www.ruanyifeng.com/blog/2013/03/tf-idf.html 写的很明了

package com.data.text.tfidf;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

public class TF_IDF {

    private  double NUM_DOCS;

    private  Map<String, Integer> idf_map;
    
    public TF_IDF(String fileName){
        idf_map = new HashMap<String, Integer>();
        File file = new File(fileName);
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new FileReader(file));
            String tempString = null;

            //第一行为Num_docs
            tempString = reader.readLine();
            NUM_DOCS = (double)Integer.parseInt(tempString);
            
            // 一次读入一行，直到读入null为文件结束
            while ((tempString = reader.readLine()) != null) {
                String[] arr = tempString.split(" : ");
                String key = arr[0];
                Integer value = Integer.parseInt(arr[1]);
                idf_map.put(key, value);
            }
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
        }
    }
    
    

    public List<Feature> cacu(Map<String, Integer> tf_map) {

        // 统计总词数
        Integer word_num_sum = 0;
        for (Entry<String, Integer> entry : tf_map.entrySet()) {
            word_num_sum += entry.getValue();
        }
        
        //计算tf-idf
        List<Feature> list_fea = new ArrayList<Feature>();
        for (Entry<String, Integer> entry : tf_map.entrySet()) {
            String word = entry.getKey();
            Integer num = entry.getValue();
            double tf = (double) num / word_num_sum;
            double idf = Math.log(NUM_DOCS / idf_map.get(word) + 1);//+1平滑 逆文档频率
            double weight = tf * idf;
            list_fea.add(new Feature(word, num, weight));
        }
        
        //根据权重排序
        Collections.sort(list_fea);

        return list_fea;
    }

    public static void main(String[] args) {
        // TODO Auto-generated method stub

    }

}


package com.data.text.tfidf;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

public class StopWord {
    
    public static Set<String> GetStopWords(){
        String fileName = "stopwords.txt";
        return readwords(fileName);
    }
    
    /**
     * 读取停用词表
     * @param fileName
     * @return
     */
    private static Set<String> readwords(String fileName){
        Set<String> set = new HashSet<String>();
        File file = new File(fileName);
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new FileReader(file));
            String tempString = null;
            
            // 一次读入一行，直到读入null为文件结束
            while ((tempString = reader.readLine()) != null) {
                set.add(tempString.trim());                
            }
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                }
            }
        }
        return set;
    }
}


package com.data.text.tfidf;

/**
 * 特征词
 * @author root
 *
 */
public class Feature implements Comparable<Feature> {
    private String word;
    private Integer num;
    private double weight;

    public Feature(String word, Integer num, double weight) {
        this.word = word;
        this.num = num;
        this.weight = weight;
    }

    public String getWord() {
        return word;
    }

    public Integer getNum() {
        return num;
    }

    public double getWeight() {
        return weight;
    }

    @Override
    public int compareTo(Feature o) {
        if(this.getWeight() == o.getWeight()){
            return 0;
        }else if(this.getWeight() > o.getWeight()){
            return -1;
        }else{
            return 1;
        }
    }
    
    public String toString(){
        return this.word + " freq: " + num + " weight: " + weight;
    }
}

 

__author__ = 'dell'

import math
import re
from operator import itemgetter


class TfIdf:
    def __init__(self, corpus_filename = None, stopword_filename = None, DEFAULT_IDF = 1.5):
        self.num_docs = 0
        self.term_num_docs = {}
        self.stopwords = []
        self.idf_default = DEFAULT_IDF

        if corpus_filename:
            corpus_file = open(corpus_filename, 'r')
            #load num of documents
            line = corpus_file.readline()
            self.num_docs = int(line)
            #read term:frequency from each subsequent line in the file
            for line in corpus_file:
                tokens = line.split(':')
                term = tokens[0].strip()
                frequency = int(tokens[1].strip())
                self.term_num_docs[term] = frequency

        if stopword_filename:
            stopword_file = open(stopword_filename)
            self.stopwords = [line.strip() for line in stopword_file]

    def get_tokens(self, str):
        return re.findall(r"<a.*?/a>|<[^\>]*>|[\w'@#]+", str.lower())

    def add_input_document(self, input):
        self.num_docs += 1
        words = set(self.get_tokens(input))
        for word in words:
            if word in self.term_num_docs:
                self.term_num_docs[word] += 1
            else:
                self.term_num_docs[word] = 1

    def get_num_docs(self):
        return self.num_docs

    def get_idf(self, term):
        if term in self.stopwords:
            return 0
        if term not in self.term_num_docs:
            return self.idf_default
        return math.log(float(1 + self.get_num_docs()) / (1 + self.term_num_docs[term]))

    def get_doc_keywords(self, curr_doc):
        tfidf = {}
        tokens = self.get_tokens(curr_doc)
        tokens_set = set(tokens)
        for word in tokens_set:
            tf = float(tokens.count(word) / len(tokens))
            idf = self.get_idf(word)
            tfidf[word] = tf * idf
        return sorted(tfidf.items(), key=itemgetter(1), reverse=True)