自然语言理解 之 统计词频

统计词频，中文字体编码格式：GB2312。

#include <iostream>
#include <fstream>
#include <algorithm>
#include <functional>
#include <string>
#include <vector>
#include <map>
#include <unordered_map>
#include <sstream>
#include <ctime>
using namespace std;

typedef long clock_t;
typedef pair<string, int> Pair_StrInt;
typedef vector<Pair_StrInt>::iterator Vec_Pair_StrInt_Itr;
#define ERROR0 cerr << "Open error !!!" << endl; exit(1);
#define ERROR1 cerr << "无法识别 !!!" << endl; exit(1);
#define Lim 100

string infile = "Ci.txt";
string outfile1 = "out1.txt";
string outfile2 = "out2.txt";
string outfile3 = "out3.txt";
string project_time = "project_time.txt";
string One_strArr[100];
string Two_strArr[100];
string Three_strArr[100];
ifstream fin;
ofstream fout;
string Text;

struct myNode {
    string Chant; // 词牌名
    string Rules; // 格式
};

bool Pair_StrInt_Cmp(const Pair_StrInt& p0, const Pair_StrInt& p1) { return (p0.second > p1.second); }
unordered_map<string, int> StrInt_Hash;

void InitText(string _infile) {
    fin.open(_infile);
    if (!fin) { ERROR0; }

    //////////////////////////////////////////////////////////////////////////
    // 将整个文件读入 string : 流迭代器
    std::ostringstream tmp;
    tmp << fin.rdbuf();
    string Text_tmp = tmp.str();
    //////////////////////////////////////////////////////////////////////////

    
    unsigned char Judge;
    string strTmp;

    int len = Text_tmp.size();
    for (int i = 0; i < len; ) {
        Judge = Text_tmp[i];
        if (Judge >= 0xB0 && Judge <= 0xF7) {
            strTmp = Text_tmp.substr(i, 2);
            i += 2;
            Text += strTmp;
        }
        else { ++i; }
    }

    fin.close();
    fin.clear();
}

// 输出到文件
void myOutput(const vector<Pair_StrInt> &StrInt_Vec, string out) {
    fout.open(out);
    if (!fout) { ERROR0; }

    vector<Pair_StrInt>::const_iterator pair_itr;
    for (pair_itr = StrInt_Vec.begin(); pair_itr != StrInt_Vec.end(); ++pair_itr) {
        fout << pair_itr->first << "\t" << pair_itr->second << endl;
    }

    fout.close();
    fout.clear();
}

// 获取一个中文字的词频
void getOneWord(string out1) {
    string strTmp;

    int str_len = Text.size();
    for (int i = 0; i < str_len; i += 2) {
        strTmp = Text.substr(i, 2);
        StrInt_Hash[strTmp] += 1;
    }
    
    vector<Pair_StrInt> StrInt_Vec(StrInt_Hash.begin(), StrInt_Hash.end());
    StrInt_Hash.clear();
    std::sort(StrInt_Vec.begin(), StrInt_Vec.end(), Pair_StrInt_Cmp);

    myOutput(StrInt_Vec, out1);

    StrInt_Vec.clear();
}

// 获取两个中文字的词频
void getTwoWord(string out2) {
    string strTmp;

    int str_len = Text.size();
    for (int i = 0; i < (str_len - 2); i += 2) {
        strTmp = Text.substr(i, 4);
        StrInt_Hash[strTmp] += 1;
    }

    vector<Pair_StrInt> StrInt_Vec(StrInt_Hash.begin(), StrInt_Hash.end());
    StrInt_Hash.clear();
    std::sort(StrInt_Vec.begin(), StrInt_Vec.end(), Pair_StrInt_Cmp);

    myOutput(StrInt_Vec, out2);

    StrInt_Vec.clear();
}

// 获取三个中文字的词频
void getThreeWord(string out3) {
    string strTmp;

    int str_len = Text.size();
    for (int i = 0; i < (str_len - 4); i += 2) {
        strTmp = Text.substr(i, 6);
        StrInt_Hash[strTmp] += 1;
    }

    vector<Pair_StrInt> StrInt_Vec(StrInt_Hash.begin(), StrInt_Hash.end());
    StrInt_Hash.clear();
    std::sort(StrInt_Vec.begin(), StrInt_Vec.end(), Pair_StrInt_Cmp);

    myOutput(StrInt_Vec, out3);

    StrInt_Vec.clear();
}

// 自动生成词
void Poetry(string _strTmp) {
    int len = _strTmp.size();
    int myRandom;
    srand((unsigned)(time(NULL)));
    for (int i = 0; i < len; ++i) {
        switch (_strTmp[i])
        {
        case '2': {
            myRandom = rand() % Lim;
            cout << Two_strArr[myRandom];
            break;
        }
        case '1': {
            myRandom = rand() % Lim;
            cout << One_strArr[myRandom];
            break;
        }
        case '3': {
            myRandom = rand() % Lim;
            cout << Three_strArr[myRandom];
            break;
        }
        case '0': {
            cout << '\n';
            break;
        }
        case '-': {
            cout << "  ";
            break;
        }
        default: {
            cout << _strTmp.substr(i, 2);
            ++i;
            break;
        }
        }
    }
    cout << endl;
}

// 生成词前的预处理
void makePoetry(string out1, string out2, string out3) {
    ifstream fin1, fin2, fin3;
    ofstream fout1, fout2, fout3;
    fin1.open(out1);
    if (!fin1) { ERROR0; }
    fin2.open(out2);
    if (!fin2) { ERROR0; }
    fin3.open(out3);
    if (!fin3) { ERROR0; }
    string strTmp;
    for (int i = 0; i < Lim; ++i) {
        getline(fin1, strTmp);
        One_strArr[i] = strTmp.substr(0, 2);
        getline(fin2, strTmp);
        Two_strArr[i] = strTmp.substr(0, 4);
        getline(fin3, strTmp);
        Three_strArr[i] = strTmp.substr(0, 6);
    }

    myNode node0;
    node0.Chant = "念奴娇";
    node0.Rules = "·220-22，12，222。22，21：222。22，22，23。22，222。0-222，23，22。22，3222。22，23，22。22，222。0";

    string strTmp0 = "---" + node0.Chant + node0.Rules;
    Poetry(strTmp0);
    //system("pause");
}

void Solve() {
    
    InitText(infile);

    ofstream fout;
    fout.open(project_time);
    clock_t myStart, myFinish;
    double totaltime;
    //////////////////////////////////////////////////////////////////////////
    myStart = clock();
    //////////////////////////////////////////////////////////////////////////
    getOneWord(outfile1);
    //////////////////////////////////////////////////////////////////////////
    getTwoWord(outfile2);
    /////////////////////////////////////////////////////////////////////////
    getThreeWord(outfile3);
    //////////////////////////////////////////////////////////////////////////

    myFinish = clock();
    totaltime = (double)(myFinish - myStart) / CLOCKS_PER_SEC;

    fout << "运行时间为： " << totaltime << " 秒。" << endl;
    fout.close();
    fout.clear();
    

    makePoetry(outfile1, outfile2, outfile3);
}

int main() {
    Solve();
    return 0;
}