【编译原理笔记】Token符号表
符号表是词法分析阶段最重要的基本原则。
// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Token is the set of lexical tokens of the Go programming language. type Token int // The list of tokens. const ( // Special tokens ILLEGAL Token = iota EOF COMMENT literal_beg // Identifiers and basic type literals // (these tokens stand for classes of literals) IDENT // main INT // 12345 FLOAT // 123.45 IMAG // 123.45i CHAR // 'a' STRING // "abc" literal_end operator_beg // Operators and delimiters ADD // + SUB // - MUL // * QUO // / REM // % AND // & OR // | XOR // ^ SHL // << SHR // >> AND_NOT // &^ ADD_ASSIGN // += SUB_ASSIGN // -= MUL_ASSIGN // *= QUO_ASSIGN // /= REM_ASSIGN // %= AND_ASSIGN // &= OR_ASSIGN // |= XOR_ASSIGN // ^= SHL_ASSIGN // <<= SHR_ASSIGN // >>= AND_NOT_ASSIGN // &^= LAND // && LOR // || ARROW // <- INC // ++ DEC // -- EQL // == LSS // < GTR // > ASSIGN // = NOT // ! NEQ // != LEQ // <= GEQ // >= DEFINE // := ELLIPSIS // ... LPAREN // ( LBRACK // [ LBRACE // { COMMA // , PERIOD // . RPAREN // ) RBRACK // ] RBRACE // } SEMICOLON // ; COLON // : operator_end keyword_beg // Keywords BREAK CASE CHAN CONST CONTINUE DEFAULT DEFER ELSE FALLTHROUGH FOR FUNC GO GOTO IF IMPORT INTERFACE MAP PACKAGE RANGE RETURN SELECT STRUCT SWITCH TYPE VAR keyword_end additional_beg // additional tokens, handled in an ad-hoc manner TILDE additional_end ) var tokens = [...]string{ ILLEGAL: "ILLEGAL", EOF: "EOF", COMMENT: "COMMENT", IDENT: "IDENT", INT: "INT", FLOAT: "FLOAT", IMAG: "IMAG", CHAR: "CHAR", STRING: "STRING", ADD: "+", SUB: "-", MUL: "*", QUO: "/", REM: "%", AND: "&", OR: "|", XOR: "^", SHL: "<<", SHR: ">>", AND_NOT: "&^", ADD_ASSIGN: "+=", SUB_ASSIGN: "-=", MUL_ASSIGN: "*=", QUO_ASSIGN: "/=", REM_ASSIGN: "%=", AND_ASSIGN: "&=", OR_ASSIGN: "|=", XOR_ASSIGN: "^=", SHL_ASSIGN: "<<=", SHR_ASSIGN: ">>=", AND_NOT_ASSIGN: "&^=", LAND: "&&", LOR: "||", ARROW: "<-", INC: "++", DEC: "--", EQL: "==", LSS: "<", GTR: ">", ASSIGN: "=", NOT: "!", NEQ: "!=", LEQ: "<=", GEQ: ">=", DEFINE: ":=", ELLIPSIS: "...", LPAREN: "(", LBRACK: "[", LBRACE: "{", COMMA: ",", PERIOD: ".", RPAREN: ")", RBRACK: "]", RBRACE: "}", SEMICOLON: ";", COLON: ":", BREAK: "break", CASE: "case", CHAN: "chan", CONST: "const", CONTINUE: "continue", DEFAULT: "default", DEFER: "defer", ELSE: "else", FALLTHROUGH: "fallthrough", FOR: "for", FUNC: "func", GO: "go", GOTO: "goto", IF: "if", IMPORT: "import", INTERFACE: "interface", MAP: "map", PACKAGE: "package", RANGE: "range", RETURN: "return", SELECT: "select", STRUCT: "struct", SWITCH: "switch", TYPE: "type", VAR: "var", TILDE: "~", } // String returns the string corresponding to the token tok. // For operators, delimiters, and keywords the string is the actual // token character sequence (e.g., for the token ADD, the string is // "+"). For all other tokens the string corresponds to the token // constant name (e.g. for the token IDENT, the string is "IDENT"). // func (tok Token) String() string { s := "" if 0 <= tok && tok < Token(len(tokens)) { s = tokens[tok] } if s == "" { s = "token(" + strconv.Itoa(int(tok)) + ")" } return s }