词法分析器——哈工大编译原理课程（一）
程序输入：从code.txt文件中读取内容
程序输出：识别出的单词序列，格式为：（种别码，属性值）
①对于关键字和运算符、分隔符来说，输出格式为（种别码，0），因为每个种别码能唯一地标识出是哪个单词
②对于标识符来说，输出格式为（id的种别码即36，在哈希桶中的位置）
③对于常量（整数、浮点数、字符串）来说，输出格式为（种别码，在数组中的位置）
1 #include<stdio.h> 2 #include<stdlib.h> 3 #include<ctype.h> 4 #include<string.h> 5 #include<math.h> 6 #include<de.h> 7 #define PRIME 211 8 #define EOS '\0' 9 #define N 1000 10 11 typedef struct idnode{ 12 char *name; 13 int kind; /*符号种类，如函数、形参等*/ 14 int type; /*类型，如int、int*等*/ 15 int firline; 16 struct idnode *next_hash; /*指向下一节点的指针*/ 17 }Identifier; 18 Identifier *SymbolTable[PRIME] = {NULL}; 19 20 char *string[500]; 21 int pstr = -1; 22 int int_num[1000]; 23 int pint = -1; 24 float real_num[1000]; 25 int preal = -1; 26 27 int lexeme_beginning = 0; /*开始指针*/ 28 int forward = 0; /*向前指针*/ 29 char *buffer = NULL; 30 31 int hashpjw(char *s); 32 int insert_hash(char *idname); 33 int token_scan(int *temp); 34 void ReadFile(); 35 char mgetchar(); 36 char *copy_token(int flag); 37 int install_id(char *token); 38 int install_num(int token); 39 int install_real(float token); 40 float matof(char *token); 41 int install_string(char *token); 42 int check_ktable(char *token); 43 char *get_substr(char *token,int start,int end); 44 void error_handle(); 45 46 int line = 1; /*行数*/ 47 int flength; 48 int error[N][2]; 49 int perr = 0; 50 char error_ch; 51 52 int main() 53 { 54 int i,temp[2]; 55 ReadFile(); 56 i = token_scan(temp); 57 if(i == 0) 58 { 59 printf("(%d,%d)\n",temp[0],temp[1]); 60 } 61 62 while(i>-2 && forward<flength) 63 { 64 lexeme_beginning = forward; 65 i = token_scan(temp); 66 if(i == 0) 67 { 68 printf("(%d,%d)\n",temp[0],temp[1]); 69 } 70 } 71 72 error_handle(); 73 return 0; 74 } 75 76 int token_scan(int *temp) 77 { 78 char ch; 79 char *token; 80 int count = 0; 81 float token_num; 82 int comment_line; 83 84 ch = mgetchar(); 85 while(ch == ' '||ch == '\t'||ch == '\n') 86 { 87 if(ch == '\n') 88 { 89 line++; 90 } 91 ch = mgetchar(); /*forward总比ch多一,ch和lexeme_begin指向同一个*/ 92 lexeme_beginning++; 93 } 94 95 if(ch == '{') 96 { 97 comment_line = line; 98 ch = mgetchar(); 99 lexeme_beginning++; 100 if(ch == '}') 101 { 102 return -1; 103 } 104 else if(ch == '*') 105 { 106 while(1) 107 { 108 while(ch != '*') 109 { 110 if(ch =='\0') 111 { 112 printf("Line %d ERROR:unterminated comment!\n",comment_line); 113 return -2; 114 } 115 ch = mgetchar(); 116 lexeme_beginning++; 117 if(ch == '\n') 118 { 119 line++; 120 } 121 } 122 ch = mgetchar(); 123 lexeme_beginning++; 124 if(ch == '\n') 125 { 126 line++; 127 } 128 if(ch == '}') 129 { 130 return -1; 131 } 132 } 133 134 } 135 else 136 { 137 while(ch != '}'&&ch != '\0'&&ch != '\n') 138 { 139 ch = mgetchar(); 140 lexeme_beginning++; 141 } 142 if(ch == '\n') 143 { 144 if(perr < N) 145 { 146 error[perr][0] = 6; 147 error[perr][1] = line; 148 perr++; 149 return 1; 150 } 151 else 152 { 153 printf("Too much Error!Please check your program!\n"); 154 return -2; 155 } 156 } 157 if(ch == '\0') 158 { 159 printf("Line %d ERROR:unterminated comment!\n",comment_line); 160 return -2; 161 } 162 return -1; 163 } 164 } 165 166 if(isalpha(ch)) /*标识符类型*/ 167 { 168 ch = mgetchar(); 169 count++; 170 while(isalpha(ch)||isdigit(ch)) 171 { 172 ch = mgetchar(); 173 count++; 174 } 175 if(count > 127) /*最大长度为127*/ 176 { 177 if(perr < N) 178 { 179 error[perr][0] = 1; 180 error[perr][1] = line; 181 perr++; 182 } 183 else 184 { 185 printf("Too much Error!Please check your program!\n"); 186 return -2; 187 } 188 /*printf("Line %d ERROR:Too long identifier!\n",line);*/ 189 } 190 191 forward--; /*forward指向other字符*/ 192 token = copy_token(0); 193 printf("%s ",token); 194 if(check_ktable(token) != -1) 195 { 196 temp[0] = check_ktable(token); /*种别*/ 197 temp[1] = 0; /*属性值*/ 198 return 0; 199 } 200 else 201 { 202 temp[0] = ID; /*种别*/ 203 temp[1] = install_id(token); /*属性值*/ 204 if(temp[1] == -1) 205 { 206 return 1; 207 } 208 return 0; 209 } 210 } 211 212 if(isdigit(ch)) /*无符号整数与无符号浮点数（含科学计数法）*/ 213 { 214 ch = mgetchar(); 215 while(isdigit(ch)) 216 { 217 ch = mgetchar(); 218 } 219 if(ch == '.') 220 { 221 ch = mgetchar(); 222 if(isdigit(ch)) 223 { 224 ch = mgetchar(); 225 while(isdigit(ch)) 226 { 227 ch = mgetchar(); 228 } 229 if(ch != 'E') 230 { 231 forward--; 232 token = copy_token(1); 233 temp[0] = REAL; 234 temp[1] = install_real(atof(token)); 235 printf("%f ",atof(token)); 236 return 0; 237 } 238 } 239 else 240 { 241 forward--; 242 if(perr < N) 243 { 244 error[perr][0] = 2; 245 error[perr][1] = line; 246 perr++; 247 return 1; 248 } 249 else 250 { 251 printf("Too much Error!Please check your program!\n"); 252 return -2; 253 } 254 /*printf("Line %d ERROR:Illeagle float num!\n",line);*/ 255 } 256 } 257 258 if(ch == 'E') 259 { 260 ch = mgetchar(); 261 if(ch == '+' || ch == '-') 262 { 263 ch = mgetchar(); 264 if(!isdigit(ch)) 265 { 266 forward--; 267 if(perr < N) 268 { 269 error[perr][0] = 2; 270 error[perr][1] = line; 271 perr++; 272 return 1; 273 } 274 else 275 { 276 printf("Too much Error!Please check your program!\n"); 277 return -2; 278 } 279 /*printf("Line %d ERROR:Illeagle float num!\n",line);*/ 280 } 281 } 282 if(isdigit(ch)) 283 { 284 ch = mgetchar(); 285 while(isdigit(ch)) 286 { 287 ch = mgetchar(); 288 } 289 forward--; 290 token = copy_token(1); 291 token_num = matof(token); /*token_num为浮点数类型*/ 292 temp[0] = REAL; 293 temp[1] = install_real(token_num); 294 printf("%f ",token_num); 295 return 0; 296 } 297 else 298 { 299 forward--; 300 if(perr < N) 301 { 302 error[perr][0] = 2; 303 error[perr][1] = line; 304 perr++; 305 return 1; 306 } 307 else 308 { 309 printf("Too much Error!Please check your program!\n"); 310 return -2; 311 } 312 } 313 } 314 315 forward--; 316 token = copy_token(1); 317 temp[0] = INT; 318 temp[1] = install_num(atoi(token)); 319 printf("%d ",atoi(token)); 320 return 0; 321 /* 322 forward--; 323 token = copy_token(1); 324 temp[0] = INT; 325 temp[1] = install_num(token); 326 */ 327 } 328 329 switch(ch) /*运算符*/ 330 { 331 case '+': 332 { 333 temp[0] = PLUS; 334 temp[1] = 0; 335 return 0; 336 break; 337 } 338 case '-': 339 { 340 temp[0] = MINUS; 341 temp[1] = 0; 342 return 0; 343 break; 344 } 345 case '/': 346 { 347 temp[0] = RDIV; 348 temp[1] = 0; 349 return 0; 350 break; 351 } 352 case '*': 353 { 354 ch = mgetchar(); 355 if(ch == '*') 356 { 357 temp[0] = EXP; 358 temp[1] = 0; 359 } 360 else 361 { 362 forward--; 363 temp[0] = MULTI; 364 temp[1] = 0; 365 } 366 return 0; 367 break; 368 } 369 case '=': 370 { 371 temp[0] = EQ; 372 temp[1] = 0; 373 return 0; 374 break; 375 } 376 case '<': 377 { 378 ch = mgetchar(); 379 if(ch == '=') 380 { 381 temp[0] = LE; 382 temp[1] = 0; 383 } 384 else if(ch == '>') 385 { 386 temp[0] = NE; 387 temp[1] = 0; 388 } 389 else 390 { 391 forward--; 392 temp[0] = LT; 393 temp[1] = 0; 394 } 395 return 0; 396 break; 397 } 398 case '>': 399 { 400 ch = mgetchar(); 401 if(ch == '=') 402 { 403 temp[0] = GE; 404 temp[1] = 0; 405 } 406 else 407 { 408 forward--; 409 temp[0] = GT; 410 temp[1] = 0; 411 } 412 return 0; 413 break; 414 } 415 case '(': 416 { 417 temp[0] = LR_BRAC; 418 temp[1] = 0; 419 return 0; 420 break; 421 } 422 case ')': 423 { 424 temp[0] = RR_BRAC; 425 temp[1] = 0; 426 return 0; 427 break; 428 } 429 case ',': 430 { 431 temp[0] = COMMA; 432 temp[1] = 0; 433 return 0; 434 break; 435 } 436 case '`': /*编码表里的P_MARK是啥？顿号？*/ 437 { 438 temp[0] = P_MARK; 439 temp[1] = 0; 440 return 0; 441 break; 442 } 443 case '.': 444 { 445 ch = mgetchar(); 446 if(ch == '.') 447 { 448 temp[0] = RANGE; 449 temp[1] = 0; 450 } 451 else 452 { 453 forward--; 454 temp[0] = F_STOP; 455 temp[1] = 0; 456 } 457 return 0; 458 break; 459 } 460 case ':': 461 { 462 ch = mgetchar(); 463 if(ch == '=') 464 { 465 temp[0] = ASSIGN; 466 temp[1] = 0; 467 } 468 else 469 { 470 forward--; 471 temp[0] = COLON; 472 temp[1] = 0; 473 } 474 return 0; 475 break; 476 } 477 case ';': 478 { 479 temp[0] = SEMIC; 480 temp[1] = 0; 481 return 0; 482 break; 483 } 484 case '^': 485 { 486 temp[0] = CAP; 487 temp[1] = 0; 488 return 0; 489 break; 490 } 491 case '[': 492 { 493 temp[0] = LS_BRAC; 494 temp[1] = 0; 495 return 0; 496 break; 497 } 498 case ']': 499 { 500 temp[0] = RS_BRAC; 501 temp[1] = 0; 502 return 0; 503 break; 504 } 505 case '\'': 506 { 507 ch = mgetchar(); 508 while(ch != '\'') 509 { 510 ch = mgetchar(); 511 if(ch == '\0') 512 { 513 if(perr < N) 514 { 515 error[perr][0] = 3; 516 error[perr][1] = line; 517 perr++; 518 return 1; 519 } 520 else 521 { 522 printf("Too much Error!Please check your program!\n"); 523 return -2; 524 } 525 /*printf("Line %d ERROR:No end of string!\n",line);*/ 526 return 1; 527 } 528 } 529 token = copy_token(1); 530 printf("%s ",token); 531 temp[0] = STRING; 532 temp[1] = install_string(token); 533 return 0; 534 break; 535 } 536 case '\0': 537 { 538 return -2; 539 break; 540 } 541 default: 542 { 543 if(perr < N) 544 { 545 error[perr][0] = 4; 546 error[perr][1] = line; 547 error_ch = ch; 548 perr++; 549 return 1; 550 } 551 else 552 { 553 printf("Too much Error!Please check your program!\n"); 554 return -2; 555 } 556 /*printf("Line %d ERROR:Illegal char '%c'\n",line,ch);*/ 557 } 558 } 559 } 560 561 void ReadFile() 562 { 563 FILE *pfile; 564 565 pfile = fopen("code.txt", "r"); 566 if (pfile == NULL) 567 { 568 printf("Open file ERROR!\n"); 569 exit(0); 570 } 571 fseek(pfile, 0, SEEK_END); 572 flength = ftell(pfile); 573 buffer = (char *)malloc((flength + 1) * sizeof(char)); 574 rewind(pfile); 575 flength = fread(buffer, 1, flength, pfile); 576 buffer[flength] = '\0'; 577 fclose(pfile); 578 } 579 580 char mgetchar() 581 { 582 char ch = *(buffer + forward); 583 forward++; 584 return ch; 585 } 586 587 char *copy_token(int flag) 588 { 589 int length = forward - lexeme_beginning; /*要拷贝的字符串长度*/ 590 char* token = (char*)calloc(sizeof(char),length+1); 591 char* pcode = buffer + lexeme_beginning; 592 int i; 593 594 if(flag == 0 && length > 63) 595 { 596 if(perr < N) 597 { 598 error[perr][0] = 5; 599 error[perr][1] = line; 600 perr++; 601 } 602 else 603 { 604 printf("Too much Error!Please check your program!\n"); 605 } 606 /*printf("Line %d Warning:identifier will be cut off!\n",line);*/ 607 for(i=0;i<63;i++) 608 { 609 token[i]=*pcode; 610 pcode++; 611 } 612 token[63]='\0'; /*加上字符串结束符*/ 613 } 614 else 615 { 616 for(i=0;i<length;i++) 617 { 618 token[i]=*pcode; 619 pcode++; 620 } 621 token[length]='\0'; /*加上字符串结束符*/ 622 } 623 return token; /*返回分配的字符数组地址 */ 624 } 625 626 int install_id(char *token) 627 { 628 int hash_ret = insert_hash(token); 629 if(hash_ret == 0) 630 { 631 return hashpjw(token); 632 } 633 printf("Line %d Error:There is a Same name!\n",hash_ret); 634 return -1; 635 636 } 637 638 float matof(char *token) 639 { 640 int i; 641 float before; 642 int after; 643 int l = strlen(token); 644 for(i = 0;i < l;i++) 645 { 646 if(token[i] == 'E') 647 { 648 before = atof(get_substr(token,0,i)); 649 if(token[i+1] == '+') 650 { 651 after = atoi(get_substr(token,i+2,l)); 652 return before*pow(10,after); 653 } 654 else if(token[i+1] == '-') 655 { 656 after = atoi(get_substr(token,i+2,l)); 657 return before*pow(10,-after); 658 } 659 else 660 { 661 after = atoi(get_substr(token,i+1,l)); 662 return before*pow(10,after); 663 } 664 } 665 } 666 return atof(token); 667 } 668 669 int install_num(int token) 670 { 671 pint++; 672 int_num[pint] = token; 673 return pint; 674 675 } 676 677 int install_real(float token) 678 { 679 preal++; 680 real_num[preal] = token; 681 return preal; 682 } 683 684 int install_string(char *token) 685 { 686 pstr++; 687 string[pstr] = token; 688 return pstr; 689 } 690 691 int check_ktable(char *token) 692 { 693 int i; 694 for(i=1;i<36;i++) 695 { 696 if(strcmp(key_table[i],token) == 0) 697 { 698 return i; 699 } 700 } 701 return -1; 702 } 703 704 char *get_substr(char *token,int start,int end) /*含左不含右*/ 705 { 706 int length = end - start; /*要拷贝的字符串长度*/ 707 char* ret = (char*)calloc(sizeof(char),length+1); 708 char* pcode = token + start; 709 int i; 710 711 for(i=0;i<length;i++) 712 { 713 ret[i]=*pcode; 714 pcode++; 715 } 716 ret[length]='\0'; /*加上字符串结束符*/ 717 return ret; /*返回分配的字符数组地址 */ 718 } 719 720 int insert_hash(char *idname) 721 { 722 int ret; 723 Identifier *id = (Identifier*)malloc(sizeof(Identifier)),*p,*pb; 724 725 ret = hashpjw(idname); 726 id->name = idname; 727 id->firline = line; 728 729 p = SymbolTable[ret]; 730 731 if(p == NULL||strcmp(p->name,idname)>0) 732 { 733 id->next_hash = p; 734 SymbolTable[ret] = id; 735 return 0; 736 } 737 738 while(p != NULL) 739 { 740 if(strcmp(p->name,idname) < 0) 741 { 742 pb = p; 743 p = p->next_hash; 744 } 745 else 746 { 747 break; 748 } 749 } 750 751 if(p == NULL) 752 { 753 id->next_hash = NULL; 754 pb->next_hash = id; 755 return 0; 756 } 757 else if(strcmp(p->name,idname)>0) 758 { 759 id->next_hash = p; 760 pb->next_hash = id; 761 return 0; 762 } 763 else 764 { 765 /*出错处理*/ 766 /*return p->firline;*/ 767 return 0; 768 } 769 } 770 771 int hashpjw(char *s) 772 { 773 char *p; 774 unsigned h = 0,g; 775 for(p = s;*p!=EOS;p=p+1) 776 { 777 h = (h<<4) + (*p); 778 if((g = (h & 0xF0000000))) 779 { 780 h = h^(g>>24); 781 h = h^g; 782 } 783 } 784 return h % PRIME; 785 } 786 787 void error_handle() 788 { 789 int i; 790 for(i = 0;i < perr;i++) 791 { 792 switch(error[i][0]) 793 { 794 case 1: 795 { 796 printf("Line %d ERROR:Too long identifier!\n",error[i][1]); 797 break; 798 } 799 case 2: 800 { 801 printf("Line %d ERROR:Illeagle float num!\n",error[i][1]); 802 break; 803 } 804 case 3: 805 { 806 printf("Line %d ERROR:No end of string!\n",error[i][1]); 807 break; 808 } 809 case 4: 810 { 811 printf("Line %d ERROR:Illegal char '%c'\n",error[i][1],error_ch); 812 break; 813 } 814 case 5: 815 { 816 printf("Line %d Warning:identifier will be cut off!\n",error[i][1]); 817 break; 818 } 819 case 6: 820 { 821 printf("Line %d ERROR:unterminated comment!\n",error[i][1]); 822 break; 823 } 824 default: 825 { 826 continue; 827 } 828 } 829 } 830 }
1 #define ID 36 2 #define INT 37 3 #define REAL 38 4 #define STRING 39 5 #define PLUS 40 6 #define MINUS 41 7 #define MULTI 42 8 #define RDIV 43 9 #define EQ 44 10 #define LT 45 11 #define GT 46 12 #define LE 47 13 #define GE 48 14 #define NE 49 15 #define LR_BRAC 50 16 #define RR_BRAC 51 17 #define COMMA 52 18 #define P_MARK 53 19 #define F_STOP 54 20 #define RANGE 55 21 #define COLON 56 22 #define ASSIGN 57 23 #define SEMIC 58 24 #define CAP 59 25 #define EXP 60 26 #define LS_BRAC 61 27 #define RS_BRAC 62 28 #define Q_MARK 63 29 30 char* key_table[36] = {"","and","array","begin","case","const","div","do","downto","else", 31 "end","file","for","function","goto","if","in","label","mod","nil","not","of","or", 32 "packed","procedure","program","record","repeat","set","then","to","type","until", 33 "var","while","with"};
一、支持类型
能识别关键字、标识符、常量（整常数、浮常数、字符串）、运算符、分隔符。能处理单行和多行注释。支持识别的各单词种别码及宏定义如下表
|
单词 |
种别码 |
宏 |
单词 |
种别码 |
宏 |
单词 |
种别码 |
宏 |
|
and |
1 |
AND |
Or |
22 |
OR |
/ |
43 |
RDIV |
|
array |
2 |
ARRAY |
Packed |
23 |
PACKED |
= |
44 |
EQ |
|
begin |
3 |
BEGIN |
procedure |
24 |
PROC |
< |
45 |
LT |
|
case |
4 |
CASE |
Program |
25 |
PROG |
> |
46 |
GT |
|
const |
5 |
CONST |
Record |
26 |
RECORD |
<= |
47 |
LE |
|
div |
6 |
DIV |
Repeat |
27 |
REPEAT |
>= |
48 |
GE |
|
do |
7 |
DO |
Set |
28 |
SET |
<> |
49 |
NE |
|
downto |
8 |
DOWNTO |
Then |
29 |
THEN |
( |
50 |
LA_BRAC |
|
else |
9 |
ELSE |
To |
30 |
TO |
) |
51 |
BR_BRAC |
|
end |
10 |
END |
Type |
31 |
TYPE |
, |
52 |
COMMA |
|
file |
11 |
FILE |
Until |
32 |
UNTIL |
、 |
53 |
P_MARK |
|
for |
12 |
FOR |
Car |
33 |
CAR |
. |
54 |
F_STOP |
|
function |
13 |
FUNC |
While |
34 |
WHILE |
.. |
55 |
RANGE |
|
goto |
14 |
GOTO |
With |
35 |
WITH |
: |
56 |
COLON |
|
if |
15 |
IF |
标识符 |
36 |
ID |
:= |
57 |
ASSIGN |
|
in |
16 |
IN |
整常数 |
37 |
INT |
; |
58 |
SEMIC |
|
label |
17 |
LABEL |
实常数 |
38 |
REAL |
^ |
59 |
CAP |
|
mod |
18 |
MOD |
字符串 |
39 |
STRING |
** |
60 |
EXP |
|
nil |
19 |
NIL |
+ |
40 |
PLUS |
[ |
61 |
LS_BRAC |
|
not |
20 |
NOT |
- |
41 |
MINUS |
] |
62 |
RS_BRAC |
|
of |
21 |
OF |
* |
|
MULTI |
‘ |
63 |
Q_MARK |
二、各种成分的定义
1、关键字
标准Pascal定义的35个关键字，表中种别码为1~35的单词
2、标识符
以字母开头的字母数字串。最长由127个字符组成，但只有前63个字符有效，多余部分将被截断；其中不允许有空格
状态转换图
3、常量
①整常数
十进制无符号整数，如11
②实常数
十进制无符号浮点数，支持科学计数法的表示形式，如11.2、11E2、11E+2、11E-2、11.3E+2
、11.3E-2等
状态转换图
③字符串
用单引号””引起来的字符序列，如'Hello, world.'等，最大长度为1024（data_struct.h定义）
4、运算符及分隔符
如上表种别码为40~63的单词
5、注释
单行注释{…}
多行注释{*…\n…\n…*}
三、错误提示
1、标识符长度超出127，提示：Line 行数 ERROR:Too long identifier!
2、标识符长度超过63，警告：Line 行数 Warning:identifier will be cut off!
3、浮点数格式不合法，如32.、32.E、32E+等，提示：Line 行数 ERROR:Illeagle float num!
4、字符串只有一个单引号，无另一个时，提示：Line 行数ERROR:No end of string!
5、程序中出现非法字符，提示：Line 行数 ERROR:Illegal char '%c'
6、注释不闭合，提示：Line 行数ERROR:unterminated comment!
四、程序说明与算法
程序指定读取文件code.txt，去掉其中的空格和注释，将其切分成一个个形如（种别码，属性值）的单词符号，输出到lex.txt中，若遇到错误，将其存入错误类型数组中，继续进行分析，这样做不会使分析到一个错误就终止程序，可以将整个分析扫描一遍，找出其中所有的错误，使用户使用起来更方便。
五、测试程序和运行截图
测试程序code.txt
program SumAverage;
{This is a program which gets the average of five numbers.}
const
NumberOfIntegers = 5;
var
A, B, C, D, E : integer;
Sum : integer;
Average,t : real;
{*test
comment*}
begin
A := 45;
B := 7;
C := 68;
D := 2;
E := 34;
Sum := A + B + C + D + E;
Average := Sum / NumberOfIntegers;
t := 11.3E+2;
writeln ('Number of integers = ', NumberOfIntegers);
writeln ('Number1 = ', A);
writeln ('Number2 = ', B);
writeln ('Number3 = ', C);
writeln ('Number4 = ', D);
writeln ('Number5 = ', E);
writeln ('Sum = ', Sum);
writeln ('Average = ', Average)
end.
运行结果截图（词法分析器在总体结构中不会打印此部分）
Lex.txt部分截图
