游戏脚本编程 文本token解析

一个数字的组成由以下几个字符

正负号 + -   小数点 .   数字 0-9

比如

3

-3

3.13

-34.2234

但是符号和小数点不会出现多次

那么识别流程用图来表示 则是

整数

 

浮点数

 

一个读取C++源文件 将内容解析成一个个单独的TOKEN的代码

代码1

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
#include <iostream>
#include <fstream>
#include <cctype>
#include <cstring>
#include <string>
#include <exception>
 
using namespace std;
 
#define SOURCE_FILE_NAME    "sourcefile.cpp"
#define DEST_FILE_NAME      "destfile.cpp"
 
// The input and output file streams.
ifstream fin;
ofstream fout;
 
 
 
bool GetToken(string& token){
    bool bRet = false;
    char ch;
 
    ch = fin.get();
    if(ch == EOF){
        return false;
    }
 
    if(isspace(ch)){
        //进入接受连续空白符(' ' '\n'等)
        while(isspace(ch)){
            token += ch;
            ch = fin.get();
        }
        fin.putback(ch);
        bRet = true;
        return bRet;
    }
 
    if(isalpha(ch)){
        while(isalpha(ch)){
            token += ch;
            ch =fin.get();
        }
        fin.putback(ch);
        bRet = true;
        return bRet;
    }
 
    if(isdigit(ch)){
        while(isdigit(ch) || ch == '.'){
            token += ch;
            ch = fin.get();
        }
        fin.putback(ch);
        bRet = true;
        return bRet;
    }
 
    if(ch == '-' || ch == '+'){
        token += ch;
        ch = fin.get();
        while(isdigit(ch) || ch == '.'){
            token += ch;
            ch = fin.get();
        }
        fin.putback(ch);
        bRet = true;
        return bRet;
    }
 
    if(ch == '<' || ch == '>'){
        token += ch;
        ch = fin.get();
        if(ch == '<' || ch == '>'){
            token += ch;
        }else{
            fin.putback(ch);
        }
        bRet = true;
        return bRet;
    }
 
 
    token += ch;
    bRet = true;
    return bRet;
}
 
 
int main(int argc, char *argv[])
{
    fin.open(SOURCE_FILE_NAME);
    if(!fin){
        cout << "Open source file error.Exit!!" << endl;
        return -1;
    }
 
    fout.open(DEST_FILE_NAME);
    if(!fout){
        cout << "Open destinaton  file error.Exit!!" << endl;
        return -1;
    }
 
    try{
        string token;
        while(GetToken(token)){
            cout << token ;//<< endl;
            token.clear();
        }
 
 
    }catch(exception& e){
        cerr << e.what() << endl;
    }
 
 
 
 
 
 
 
    fin.close();
    fout.close();
    cout << "Hello World!"<<endl;
    return 0;
}

　测试文件

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
293048 24 895523
3.14159
235
        253
          52435 345
 
           459245
 
 22 .5 .35 2.0
  
1
 0.0
  1.0
   0
 
    02345
         
    63246 0.2346
    34.0

　代码2

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
#include <iostream>
#include <fstream>
#include <exception>
#include <queue>
using namespace std;
 
#define IN_FILE_NAME    "SourceFile.cpp"
#define OUT_FILE_NAME    "DestinationFile.cpp"
 
enum STATE{
    state_init = 0,
    state_int,
    state_float,
    state_error
};
 
class FileParse{
public:
    FileParse(const string& infileName,const string& outfileName){
        fin_.open(infileName);
        fout_.open(outfileName);
    }
    ~FileParse(){
        if(fin_.is_open())
            fin_.close();
        if(fout_.is_open())
            fout_.close();
    }
 
    bool ParseToTokens(){
        STATE state = state_init;
        bool isFinish = false;
        string token;
 
        if(linestr_.empty())
            return false;
 
        for(size_t i = 0;i<linestr_.size();++i){
            char currentChar = linestr_[i];
            if(currentChar == '\0')
                break;
 
            switch(state){
            case state_init:
                if(isspace(currentChar)){
                    continue;
                }else if(isdigit(currentChar)){
                    state = state_int;
                    token += currentChar;
                    continue;
                }else if(currentChar == '.'){
                    state = state_float;
                    token += currentChar;
                    continue;
                }else{
                    state = state_error;
                    break;
                }
            case state_int:
                if(isdigit(currentChar)){
                    state = state_int;
                    token += currentChar;
                    continue;
                }else if(currentChar == '.'){
                    state = state_float;
                    token += currentChar;
                    continue;
                }else if(isspace(currentChar)){
                    isFinish = true;
                    break;
                }else{
                    state = state_error;
                    break;
                }
 
            case state_float:
                if(isdigit(currentChar)){
                    state = state_int;
                    token += currentChar;
                    continue;
                }else if(isspace(currentChar)){
                    isFinish = true;
                    break;
                }else{
                    state = state_error;
                    break;
                }
 
            case state_error:
                break;
            }
 
            if(isFinish ){
                cout << token <<endl;
                token.clear();
                isFinish = false;
                state = state_init;
            }
         }
 
 
        return true;
    }
 
    bool run(){
        try{
            if(!fin_.is_open() || !fout_.is_open()) {
                throw runtime_error("open file is null");
            }
            while(1){
                if (fin_.eof())
                    break;
                linestr_.clear();
                getline(fin_,linestr_);
                linestr_ += '\n';
                ParseToTokens();
            }
 
        }catch(exception& e){
            cerr << e.what() << endl;
            return false;
        }
 
 
 
    }
 
private:
    string linestr_;
    queue<string> vecToken_;
    ifstream fin_;
    ofstream fout_;
};
 
 
int main(int argc, char *argv[])
{
    FileParse a(IN_FILE_NAME,OUT_FILE_NAME);
    a.run();
    return 0;
}

　　显示结果

 

代码3 新增字符串的识别解析

　　测试文本

　　显示结果

 

posted on   itdef  阅读(716)  评论(0)  编辑  收藏  举报

编辑推荐：
· go语言实现终端里的倒计时
· 如何编写易于单元测试的代码
· 10年+ .NET Coder 心语，封装的思维：从隐藏、稳定开始理解其本质意义
· .NET Core 中如何实现缓存的预热？
· 从 HTTP 原因短语缺失研究 HTTP/2 和 HTTP/3 的设计差异
阅读排行：
· 分享一个免费、快速、无限量使用的满血 DeepSeek R1 模型，支持深度思考和联网搜索！
· 基于 Docker 搭建 FRP 内网穿透开源项目（很简单哒）
· ollama系列1：轻松3步本地部署deepseek，普通电脑可用
· 按钮权限的设计及实现
· 25岁的心里话

导航

< 2025年3月 >
日 一 二 三 四 五 六
23 24 25 26 27 28 1
2 3 4 5 6 7 8
9 10 11 12 13 14 15
16 17 18 19 20 21 22
23 24 25 26 27 28 29
30 31 1 2 3 4 5

统计

点击右上角即可分享
微信分享提示
AI IDE Trae