c++实现文本中英文单词和汉字字符的统计

源代码下载：http://download.csdn.net/detail/nuptboyzhb/4987141

1.统计文本中汉字的频数，为后续的文本分类做基础。对于汉字的统计，需要判断读取的是否为汉字。源代码如下：

[C++ code]

/*
 *@author:郑海波 http://blog.csdn.net/NUPTboyZHB
 *参考：实验室小熊
 *注：有删改
 */
#pragma warning(disable:4786)
#include <iostream>
#include <vector>
#include <fstream>
#include <string>
#include <map>
#include <queue>
#include <ctime>
using namespace std;
void topK(const int &K)
{
    double t=clock();

    ifstream infile("test.txt");
    if (!infile)
        cout<<"can not open file"<<endl;

    string s="";
    map<string,int>wordcount;
    unsigned char temp[2];
    while(true)//国标2312
    {
        infile>>temp[0];
        if(infile.eof()) break;
        if (temp[0]>=0xB0)//GB2312下的汉字，最小是0XB0
        {
            s+=temp[0];
            infile>>temp[1];
            s+=temp[1];
        }
        else//非汉字字符不统计
        {
            s="";
            continue;
        }
        wordcount[s]++;
        s="";
    }
    cout<<"单词种类："<<wordcount.size()<<endl;
    //优先队列使用小顶堆，排在前面的数量少，使用">";
    priority_queue< pair< int,string >,vector< pair< int,string > >,greater< pair< int,string> > > queueK;
    for (map<string,int>::iterator iter=wordcount.begin(); iter!=wordcount.end(); iter++)
    {
        queueK.push(make_pair(iter->second,iter->first));
        if(queueK.size()>K)
            queueK.pop();
    }
    pair<int,string>tmp;
    //将排在后面的数量少，排在前面的数量多
    priority_queue< pair< int,string >,vector< pair< int,string > >,less< pair< int,string> > > queueKless;
    while (!queueK.empty())
    {
        tmp=queueK.top();
        queueK.pop();
        queueKless.push(tmp);
    }
    while(!queueKless.empty())
    {
        tmp=queueKless.top();
        queueKless.pop();
        cout<<tmp.second<<"\t"<<tmp.first<<endl;
    }
    cout<<"< Elapsed Time: "<<(clock()-t)/CLOCKS_PER_SEC<<" s>"<<endl;
}

int main()
{
    int k=0;
    cout<<"http://blog.csdn.net/NUPTboyZHB\n";
    while (true)
    {
        cout<<"查看前K个频率最高的汉字，K=";
        cin>>k;
        if(k<=0)break;
        topK(k);
    }
    return 0;
}

[图1]

2.统计英文单词的出现频率。这比统计汉字更加的容易，因为单词和单词之间是用空格分开的，所以，直接将单词保存到string中即可。

[c++ code]

/*
 *@author:郑海波 http://blog.csdn.net/NUPTboyZHB
 *参考：实验室小熊
 *注：有删改
 */
#pragma warning(disable:4786)
#include <iostream>
#include <vector>
#include <fstream>
#include <string>
#include <map>
#include <queue>
#include <ctime>
using namespace std;
void topK(const int &K)
{
    double t=clock();

    ifstream infile;
    infile.open("test.txt");
    if (!infile)
        cout<<"can not open file"<<endl;
    string s;
    map<string,int>wordcount;

    while(true)
    {
        infile>>s;
        if(infile.eof()) break;
        wordcount[s]++;
    }
    cout<<"单词种类："<<wordcount.size()<<endl;
    //优先队列使用小顶堆，排在前面的数量少，使用">";
    priority_queue< pair< int,string >,vector< pair< int,string > >,greater< pair< int,string> > > queueK;
    for (map<string,int>::iterator iter=wordcount.begin(); iter!=wordcount.end(); iter++)
    {
        queueK.push(make_pair(iter->second,iter->first));
        if(queueK.size()>K)
            queueK.pop();
    }
    pair<int,string>tmp;
    priority_queue< pair< int,string >,vector< pair< int,string > >,less< pair< int,string> > > queueKless;
    while (!queueK.empty())
    {
        tmp=queueK.top();
        queueK.pop();
        queueKless.push(tmp);
    }
    while(!queueKless.empty())
    {
        tmp=queueKless.top();
        queueKless.pop();
        cout<<tmp.second<<"\t"<<tmp.first<<endl;
    }
    cout<<"< Elapsed Time: "<<(clock()-t)/CLOCKS_PER_SEC<<" >"<<endl;
}
int main()
{
    int k=0;
    cout<<"http://blog.csdn.net/NUPTboyZHB\n";
    while (true)
    {
        cout<<"PUT IN K: ";
        cin>>k;
        if(k<=0)break;
        topK(k);
    }
    return 0;
}

[图2]

参考：实验室小熊