感知机算法C++实现

　　本节介绍感知机分类器的C++实现，其分类器实现框架会在后续的一些算法实现都会使用到。

　　1、类的设计

　　感知机类设计如下：

#pragma once
#include <iostream>
#include <fstream>
#include <iomanip>
#include <string>
#include <vector>
#include <set>
#include <map>
#include <algorithm>
#include <cmath>
#include <ctime>
using namespace std;

struct sparse_feat   #特征稀疏表示数据结构
{
    vector<int> id_vec;
    vector<float> value_vec;
};

class Per
{
private:
    vector<sparse_feat> samp_feat_vec; #存储数据集特征
    vector<int> samp_class_vec;        #存储数据集的类别
    int feat_set_size;     #数据集的特征数
    int class_set_size;    #数据集的类别数
    vector< vector<float> > omega;  #模型参数
     
public:
    Per();
    ~Per();
    void save_model(string model_file);
    void load_model(string model_file);
    void load_training_file(string training_file);
    void init_omega();
    
    int train_online(int criter, int max_loop, double loss_thrd, float learn_rate, float lambda, int avg);   #在线更新，主函数
    vector<float> calc_score(sparse_feat &samp_feat);
    vector<float> calc_neg_score(sparse_feat &samp_feat);
    vector<float> score_to_prb(vector<float> &score);
    int score_to_class(vector<float> &score);
    
    float classify_testing_file(string testing_file, string output_file, int output_format);

private:
    void read_samp_file(string samp_file, vector<sparse_feat> &samp_feat_vec, vector<int> &samp_class_vec);
    void update_online_per(int samp_class, sparse_feat &samp_feat, float learn_rate);   #感知机更新侧率
　　 void calc_loss_per(double *loss, float *acc);                                       #感知机损失函数计算
    float calc_acc(vector<int> &test_class_vec, vector<int> &pred_class_vec);    
    sparse_feat product_pair_feature(const sparse_feat &samp_feat);
    
    vector<string> string_split(string terms_str, string spliting_tag);

};

 
　　本文采用稀疏表示的数据结构存储特征，用结构体sparse_feat表示特征，其中id_vec存储索引号的向量，value_vec存储特征值的向量，两者一一对应。目前主流的机器学习工具包均采用稀疏特征的存储方式。

　　2.实现代码：

/********************************************************************
* Linear Discriminant Function Classifier V0.10
*********************************************************************/

#include "Per.h"

Per::Per()
{
}

Per::~Per()
{
}

void Per::save_model(string model_file)
{ 
    cout << "Saving model..." << endl;
    ofstream fout(model_file.c_str());
    for (int k = 0; k < feat_set_size; k++) {
        for (int j = 0; j < class_set_size; j++) {
            fout << omega[k][j] << " ";
        }
        fout << endl;
    }
    fout.close();
}


void Per::load_model(string model_file)
{
    cout << "Loading model..." << endl;
    omega.clear();
    ifstream fin(model_file.c_str());
    if(!fin) {
        cerr << "Error opening file: " << model_file << endl;
        exit(0);
    }
    string line_str;
    while (getline(fin, line_str)) 
    {
        vector<string> line_vec = string_split(line_str, " ");
        vector<float>  line_omega;
        for (vector<string>::iterator it = line_vec.begin(); it != line_vec.end(); it++) 
        {
            float weight = (float)atof(it->c_str());
            line_omega.push_back(weight);
        }
        omega.push_back(line_omega);
    }
    fin.close();
    feat_set_size = (int)omega.size();
    class_set_size = (int)omega[0].size();
}


void Per::read_samp_file(string samp_file, vector<sparse_feat> &samp_feat_vec, vector<int> &samp_class_vec) {
    ifstream fin(samp_file.c_str());
    if(!fin) 
    {
        cerr << "Error opening file: " << samp_file << endl;
        exit(0);
    }
    string line_str;
    while (getline(fin, line_str)) 
    {
        size_t class_pos = line_str.find_first_of("\t");
        int class_id = atoi(line_str.substr(0, class_pos).c_str());
        samp_class_vec.push_back(class_id);
        string terms_str = line_str.substr(class_pos+1);
        sparse_feat samp_feat;
        samp_feat.id_vec.push_back(0); // bias
        samp_feat.value_vec.push_back(1); // bias
        if (terms_str != "") {
            vector<string> fv_vec = string_split(terms_str, " ");
            for (vector<string>::iterator it = fv_vec.begin(); it != fv_vec.end(); it++)
            {
                size_t feat_pos = it->find_first_of(":");
                int feat_id = atoi(it->substr(0, feat_pos).c_str());
                float feat_value = (float)atof(it->substr(feat_pos+1).c_str());
                samp_feat.id_vec.push_back(feat_id);
                samp_feat.value_vec.push_back(feat_value);
            }
        }
        samp_feat_vec.push_back(samp_feat);
    }
    fin.close();
}


void Per::load_training_file(string training_file)
{
    cout << "Loading training data..." << endl;
    read_samp_file(training_file, samp_feat_vec, samp_class_vec);
    feat_set_size = 0;
    class_set_size = 0;
    for (size_t i = 0; i < samp_class_vec.size(); i++) 
    {
        if (samp_class_vec[i] > class_set_size) 
        {
            class_set_size = samp_class_vec[i];
        }
        if (samp_feat_vec[i].id_vec.back() > feat_set_size) {
            feat_set_size = samp_feat_vec[i].id_vec.back();
        }    
    }
    class_set_size += 1;
    feat_set_size += 1;
}

void Per::init_omega()
{
    //float int_value = 0.0;
    float int_value = (float)1/class_set_size;
    for (int i = 0; i < feat_set_size; i++)
    {
        vector<float> temp_vec(class_set_size, int_value);
        omega.push_back(temp_vec);
    }
}

// Stochastic Gradient Descent (SGD) optimization for the criteria of Perceptron (PER), Cross Entropy (CE), Leat Mean Square (LMS)
int Per::train_online(int criter, int max_loop, double loss_thrd, float learn_rate, float lambda, int avg)
{
    int id = 0;
    double loss = 0.0;
    double loss_pre = 0.0;
    vector< vector<float> > omega_sum(omega);
    while (id <= max_loop*(int)samp_class_vec.size()) 
    {
        // check loss value
        if (id%samp_class_vec.size() == 0) {
            int loop = id/(int)samp_class_vec.size();
            double loss = 0.0;
            float acc = 0.0;
        
            calc_loss_per(&loss, &acc);
        
            cout.setf(ios::left);
            cout << "Iter: " << setw(8) << loop << "Loss: " << setw(18) << loss << "Acc: " << setw(8) << acc << endl;
            if ((loss_pre - loss) < loss_thrd && loss_pre >= loss && id != 0)
            {
                cout << "Reaching the minimal loss value decrease!" << endl;
                break;
            }
            loss_pre = loss;
        }
        // update omega
        int r = (int)(rand()%samp_class_vec.size());
        //int r = (int)i%samp_class_vec.size();
        sparse_feat samp_feat = samp_feat_vec[r];
        int samp_class = samp_class_vec[r];
        update_online_per(samp_class, samp_feat, learn_rate);
        if (avg == 1 && id%samp_class_vec.size() == 0) 
        {
            for (int i = 0; i < feat_set_size; i++)
            {
                for (int j = 0; j < class_set_size; j++)
                {
                    omega_sum[i][j] += omega[i][j];
                }
            }            
        }
        id++;
    }
    if (avg == 1) {
        for (int i = 0; i < feat_set_size; i++) 
        {
            for (int j = 0; j < class_set_size; j++)
            {
                omega[i][j] = (float)omega_sum[i][j] / id;
            }
        }        
    }
    return 1;
}

void Per::update_online_per(int samp_class, sparse_feat &samp_feat, float learn_rate)
{
    vector<float> score = calc_score(samp_feat);
    int pred_class = score_to_class(score);
    if (samp_class != pred_class) 
    {          
        for (size_t k = 0; k < samp_feat.id_vec.size(); k++) 
        {
            int feat_id = samp_feat.id_vec[k];
            float feat_value = samp_feat.value_vec[k];
            omega[feat_id][pred_class] -= learn_rate * feat_value;
            omega[feat_id][samp_class] += learn_rate * feat_value;
        }
    }
}



void Per::calc_loss_per(double *loss, float *acc)
{
    double loss_value = 0.0;
    int err_num = 0;
    for (size_t i = 0; i < samp_class_vec.size(); i++)
    {
        int samp_class = samp_class_vec[i];
        sparse_feat samp_feat = samp_feat_vec[i];
        vector<float> score = calc_score(samp_feat);
        int pred_class = score_to_class(score);
        if (pred_class != samp_class) 
        {
            err_num++;
            loss_value += (score[pred_class] - score[samp_class]);
        }
    }
    *acc = 1 - (float)err_num / samp_class_vec.size();
    *loss = loss_value / samp_class_vec.size();
}



vector<float> Per::calc_score(sparse_feat &samp_feat)
{
    vector<float> score(class_set_size, 0);
    for (int j = 0; j < class_set_size; j++)
    {
        for (size_t k = 0; k < samp_feat.id_vec.size(); k++)
        {
            int feat_id = samp_feat.id_vec[k];
            float feat_value = samp_feat.value_vec[k];
            score[j] += omega[feat_id][j] * feat_value;
        }
    }
    return score;
}

vector<float> Per::calc_neg_score(sparse_feat &samp_feat)
{
    vector<float> score(class_set_size, 0);
    for (int j = 0; j < class_set_size; j++) 
    {
        for (size_t k = 0; k < samp_feat.id_vec.size(); k++)
        {
            int feat_id = samp_feat.id_vec[k];
            float feat_value = 1-samp_feat.value_vec[k];
            score[j] += omega[feat_id][j] * feat_value;
        }
    }
    return score;

}
vector<float> Per::score_to_prb(vector<float> &score)
{   
    vector<float> prb(class_set_size, 0);
    for (int i = 0; i < class_set_size; i++)
    {
        float delta_prb_sum = 0.0;
        for (int j = 0; j < class_set_size; j++)
        {
            delta_prb_sum += exp(score[j] - score[i]);
        }
        prb[i] = 1 / delta_prb_sum;
    }
    return prb;
}

int Per::score_to_class(vector<float> &score)
{
    int pred_class = 0;    
    float max_score = score[0];
    for (int j = 1; j < class_set_size; j++)
    {
        if (score[j] > max_score) {
            max_score = score[j];
            pred_class = j;
        }
    }
    return pred_class;
}

float Per::classify_testing_file(string testing_file, string output_file, int output_format)
{
    cout << "Pair Classifying testing file..." << endl;
    vector<sparse_feat> test_feat_vec;
    vector<int> test_class_vec;
    vector<int> pair_test_class_vec;
    vector<int> pred_class_vec;
    vector<int> pair_pred_class_vec;
    read_samp_file(testing_file, test_feat_vec, test_class_vec);
    ofstream fout(output_file.c_str());
    for (size_t i = 0; i < test_class_vec.size(); i++) 
    {
        int samp_class = test_class_vec[i];
        pair_test_class_vec.push_back(1-test_class_vec[i]);

        sparse_feat samp_feat = test_feat_vec[i];
        sparse_feat pair_samp_feat=product_pair_feature(test_feat_vec[i]);

        vector<float> pred_score = calc_score(samp_feat);            
        int pred_class = score_to_class(pred_score);

        vector<float> pair_pred_score=calc_score(pair_samp_feat);
        int pair_pred_class=score_to_class(pair_pred_score);

        pred_class_vec.push_back(pred_class);
        

        if (pred_class==pair_pred_class)
        {
            vector<float> pred_prb = score_to_prb(pred_score);
            vector<float> pair_pred_prb=score_to_prb(pair_pred_score);
            float max_score=*(max_element(pred_prb.begin(),pred_prb.end()));
            float pair_max_score=*(max_element(pair_pred_prb.begin(),pair_pred_prb.end()));
            if (pair_max_score-max_score>0.2)
                pair_pred_class=1-pair_pred_class;
        }

        pair_pred_class_vec.push_back(pair_pred_class);

        fout << pred_class << "\t";
        if (output_format == 1) {
            for (int j = 0; j < class_set_size; j++) {
                fout << j << ":" << pred_score[j] << ' '; 
            }        
        }
        else if (output_format == 2)
        {
            vector<float> pred_prb = score_to_prb(pred_score);
            for (int j = 0; j < class_set_size; j++)
            {
                fout << j << ":" << pred_prb[j] << ' '; 
            }
        }
        fout << endl;        
    }
    fout.close();
    float acc = calc_acc(test_class_vec, pred_class_vec);
    float pair_acc=calc_acc(pair_test_class_vec,pair_pred_class_vec);
    cout<<"Orgional acc is :"<<acc<<endl;
    cout<<"Pair acc is :"<<pair_acc<<endl;
    return acc;
}

float Per::calc_acc(vector<int> &test_class_vec, vector<int> &pred_class_vec)
{
    size_t len = test_class_vec.size();
    if (len != pred_class_vec.size()) 
    {
        cerr << "Error: two vectors should have the same lenght." << endl;
        exit(0);
    }
    int err_num = 0;
    for (size_t id = 0; id != len; id++)
    {
        if (test_class_vec[id] != pred_class_vec[id])
        {
            err_num++;
        }
    }
    return 1 - ((float)err_num) / len;
}


vector<string> Per::string_split(string terms_str, string spliting_tag)
{
    vector<string> feat_vec;
    size_t term_beg_pos = 0;
    size_t term_end_pos = 0;
    while ((term_end_pos = terms_str.find_first_of(spliting_tag, term_beg_pos)) != string::npos) 
    {
        if (term_end_pos > term_beg_pos) 
        {
            string term_str = terms_str.substr(term_beg_pos, term_end_pos - term_beg_pos);
            feat_vec.push_back(term_str);
        }
        term_beg_pos = term_end_pos + 1;
    }
    if (term_beg_pos < terms_str.size()) 
    {
        string end_str = terms_str.substr(term_beg_pos);
        feat_vec.push_back(end_str);
    }
    return feat_vec;
}

sparse_feat Per::product_pair_feature(const sparse_feat &samp_feat)
{
    sparse_feat pair_sparse_feat;
    for (int i=0;i!=samp_feat.id_vec.size();i++)
    {
        int feat_id=samp_feat.id_vec[i];
        pair_sparse_feat.id_vec.push_back(feat_id);
        float feat_value=samp_feat.value_vec[i];
        pair_sparse_feat.value_vec.push_back(2-feat_value);

    }

    return pair_sparse_feat;

}

　　3.代码说明
　　上述算法是后续代码的框架，比如load_model(), save_model(), update_online()等都是通用的，只是在不同的分类器，迭代侧率与损失函数不一样而已(算法关键)，在感知机分类器我们迭代策略在使用update_online_per()函数，在计算损失函数使用calc_loss_per()函数。因此，你可以根据自己的需求，在上述算法框架上增加不同的准则策略。

　　4.模型的训练与预测

　　感知模型训练主函数Demo

　　

View Code 

#include <cstdlib>
#include <iostream>
#include <cstring>
#include "Per.h"

using namespace std;


void print_help() 
{
    cout << "\nPer training module, " << VERSION << ", " << VERSION_DATE << "\n\n"
        << "usage: per_train [options] training_file model_file [pre_model_file]\n\n"
        << "options: -h        -> help\n"
        << "         -n int    -> maximal iteration loops (default 200)\n"
        << "         -m double -> minimal loss value decrease (default 1e-03)\n"
        << "         -r double -> regularization parameter lambda of gaussian prior (default 0)\n"        
        << "         -l float  -> learning rate (default 1.0)\n"
        << "         -a        -> 0: final weight (default)\n"
        << "                   -> 1: average weights of all iteration loops\n"
        << "         -u [0,1]  -> 0: initial training model (default)\n"
        << "                   -> 1: updating model (pre_model_file is needed)\n" 
        << endl;
}

void read_parameters(int argc, char *argv[], char *training_file, char *model_file, 
                        int *criter, int *max_loop, double *loss_thrd, float *learn_rate, float *lambda,
                        int *avg, int *update, char *pre_model_file) {
    // set default options
    *criter = 0;
    *max_loop = 200;
    *loss_thrd = 1e-3;
    *learn_rate = 1.0;
    *lambda = 0.0;
    *avg = 0;
    *update = 0;
    int i;
    for (i = 1; (i<argc) && (argv[i])[0]=='-'; i++) {
        switch ((argv[i])[1]) {
            case 'h':
                print_help();
                exit(0);
            case 'c':
                *criter = atoi(argv[++i]);
                break;
            case 'n':
                *max_loop = atoi(argv[++i]);
                break;
            case 'm':
                *loss_thrd = atof(argv[++i]);
                break;
            case 'l':
                *learn_rate = (float)atof(argv[++i]);
                break;
            case 'r':
                *lambda = (float)atof(argv[++i]);
                break;
            case 'a':
                *avg = atoi(argv[++i]);
                break;
            case 'u':
                *update = atoi(argv[++i]);
                break;
            default:
                cout << "Unrecognized option: " << argv[i] << "!" << endl;
                print_help();
                exit(0);
        }
    }
    
    if ((i+1)>=argc) {
        cout << "Not enough parameters!" << endl;
        print_help();
        exit(0);
    }
    strcpy (training_file, argv[i]);
    strcpy (model_file, argv[i+1]);
    if (*update) {
        if ((i+2)>=argc) {
            cout << "Previous model file is needed in update mode!" << endl;
            print_help();
            exit(0);
        }
        strcpy (pre_model_file, argv[i+2]);
    }
}

int per_train(int argc, char *argv[])
{
    char training_file[200];
    char model_file[200];
    int criter;
    int max_loop;
    double loss_thrd;
    float learn_rate;
    float lambda;
    int avg;
    int update;
    char pre_model_file[200];
    read_parameters(argc, argv, training_file, model_file, &criter, &max_loop, &loss_thrd, &learn_rate, &lambda, &avg, &update, pre_model_file);
    
    Per per;
    per.load_training_file(training_file);
    if (update) {
        per.load_model(pre_model_file);
    }
    else {
        per.init_omega();    
    }
    per.train_online(criter, max_loop, loss_thrd, learn_rate, lambda, avg);
    per.save_model(model_file);
    return 0;
}

int main(int argc, char *argv[])
{
    return per_train(argc, argv);
}

　　感知机模型预测主函数Demo

View Code 

View Code 

#include <cstdlib>
#include <iostream>
#include <cstring>
#include "Per.h"

using namespace std;


void print_help()
 {
    cout << "\nOpenPR-LDF classification module, " << VERSION << ", " << VERSION_DATE << "\n\n"
        << "usage: ldf_classify [options] testing_file model_file output_file\n\n"
        << "options: -h        -> help\n"
        << "         -f [0..2] -> 0: only output class label (default)\n"
        << "                   -> 1: output class label with log-likelihood (weighted sum)\n"
        << "                   -> 2: output class label with soft probability\n"
        << endl;
}

void read_parameters(int argc, char *argv[], char *testing_file, char *model_file, 
                        char *output_file, int *output_format) 
    {
    // set default options
    *output_format = 0;
    int i;
    for (i = 1; (i<argc) && (argv[i])[0]=='-'; i++) 
    {
        switch ((argv[i])[1]) 
        {
            case 'h':
                print_help();
                exit(0);
            case 'f':
                *output_format = atoi(argv[++i]);
                break;
            default:
                cout << "Unrecognized option: " << argv[i] << "!" << endl;
                print_help();
                exit(0);
        }
    }
    
    if ((i+2)>=argc)
     {
        cout << "Not enough parameters!" << endl;
        print_help();
        exit(0);
    }
    strcpy(testing_file, argv[i]);
    strcpy(model_file, argv[i+1]);
    strcpy(output_file, argv[i+2]);
}

int per_classify(int argc, char *argv[])
{
    char testing_file[200];
    char model_file[200];
    char output_file[200];
    int output_format;
    read_parameters(argc, argv, testing_file, model_file, output_file, &output_format);
    Per per;
    per.load_model(model_file);
    float acc = per.classify_testing_file(testing_file, output_file, output_format);
    cout << "Accuracy: " << acc << endl;
    return 0;
}

int main(int argc, char *argv[])
{
    return per_classify(argc, argv);
}