python实现： protobuf解释器
之前项目为了自动化，所以写一个protobuf的解释器，用来生成项目所需的格式。
当然现在通过以下链接的指导，跳过手工分析，直接生成代码了。
https://developers.google.com/protocol-buffers/docs/reference/cpp-generated
这次文档主要是描述如何分析protobuf格式，以及如何收集需要的符号。
使用python 2.7脚本进行文本的处理。
程序分成4个模块：
expression: 格式的解析
symbol：在protobuf中定义的message等对象以及它们的层次结构，在这里已经看不见protobuf的样子了。
typecollection：基础类型定义和收集message等对象。
builder：遍历symbol，根据需要创建适合的输出文件。typecollection起到索引的作用。这次就不演示了。
1 测试用protobuf文件。（来源于google示例）
package tutorial;
message Person {
required string name = 1;
required int32 id = 2 ;
optional string email = 3;
enum PhoneType {
MOBILE = 0;
HOME = 1;
WORK = 2;
}
message PhoneNumber {
required string number = 1;
optional PhoneType type = 2 [default = HOME];
}
repeated PhoneNumber phone = 4;
}
message AddressBook {
repeated Person person = 1;
}
2 expression实现---最简单的扫描方法，分析每一个word。
# -*- coding: UTF-8 -*-
# pb_expression.py import sys import os import string import shutil import io import pb_symbol class StringBuffer(object): def __init__(self,src): self.src = src; pass;
def __del__(self): self.buf = None; pass; def OpenFile(self): self.Data = open(self.src).read() pass; class Expression(object): desc_set = set(['required','optional','repeated']) b_char_set = set(['A','B','C','D','E' ,'F','G','H','I','J' ,'K','L','M','N','O' ,'P','Q','R','S','T' ,'U','V','W','X','Y','Z']) l_char_set = set (['a','b','c','d','e' ,'f','g','h','i','j' ,'k','l','m','n','o' ,'p','q','r','s','t' ,'u','v','w','x','y','z']) digit_set = set([0,1,2,3,4,5,6,7,8,9]) equals_char = '=' space_char = ' ' openbrace_char = '{' closebrace_char = '}' semicolon_char = ';' tab_char = chr(9) newline_char = chr(10) return_char = chr(13) slash_char = chr(47) ctl_char_set = set([openbrace_char,closebrace_char,semicolon_char,equals_char,'\n','\r','\t','=',';',space_char]) empty_char_set = set ([space_char,tab_char,newline_char,return_char]) symbol_char_set = b_char_set | l_char_set | digit_set all_char_set = symbol_char_set | ctl_char_set def backup(self): return self.index; def restore(self,prevIndex): self.index = prevIndex;
def forwardChar(self): if(self.index < self.count): self.index = self.index +1 def backChar(self): if(self.index > 0): self.index = self.index -1 def getchar(self): if( self.index < self.count): char = self.Buf.Data[self.index] self.forwardChar() return char return None; def skipComment(self): bkIndex = self.backup(); while 1: char = self.getchar() next_char = self.getchar() if(char != self.slash_char or next_char != self.slash_char): self.restore(bkIndex) return; while 1: char = self.getchar() if(char == None): self.restore(bkIndex) return; if(char == self.newline_char): return; def getSpecialChar(self,currentchar): while 1: self.skipComment() char = self.getchar(); if(char == None): break; else: if(char == currentchar): break; return char; def getVisibleChar(self): while 1: self.skipComment() char = self.getchar(); if(char is None): break; else: if(char not in self.empty_char_set): break; return char; def getNextword(self): word = None got1st = 0 while 1: self.skipComment() char = self.getchar() if(char == None): break; if(got1st == 0): if(char not in self.ctl_char_set): word = char got1st = 1 else: if(char in self.ctl_char_set): self.backChar() break; else: word = word + char return word; def do_enum_item(self,pbEnum): memText = self.getNextword(); self.getSpecialChar(self.equals_char); memValue = self.getNextword(); self.getSpecialChar(self.semicolon_char); pbEnum.append_Member(memText,memValue) def do_enum_proc(self): symbol = self.getNextword(); pbEnum = pb_symbol.PBEnum(symbol) while 1: currentIndex = self.backup() word = self.getNextword(); if(word == None): break; self.restore(currentIndex) self.do_enum_item(pbEnum) end_char_Index = self.backup(); char = self.getVisibleChar(); if(char == self.closebrace_char): break; else: self.restore(end_char_Index); self.symbol.append_enum(pbEnum) def do_message_proc(self): symbol = self.getNextword(); pbMsg = pb_symbol.PBMessage(symbol) while 1: currentIndex = self.backup() word = self.getNextword(); if(word == None): break; if(word in self.token_set): subSymbol = pb_symbol.Symbol(self.symbol.tpDict,self.symbol.entity_full_path,False); subSymbol.update_namespace(symbol); self.restore(currentIndex); subExp = Expression(self.Buf,subSymbol); subExp.index = self.index; subExp.do_expression(); self.index = subExp.index self.symbol.append_symbol(subSymbol) pbMsg.enableSymbol = 1 else: if(word in self.desc_set): memType = self.getNextword(); memText = self.getNextword(); pbMsg.append_Member(word,memType,memText) self.getSpecialChar(self.semicolon_char); end_char_Index = self.backup(); char = self.getVisibleChar(); if(char == self.closebrace_char): break; else: self.restore(end_char_Index); self.symbol.append_message(pbMsg) def do_import_proc(self): self.getSpecialChar(self.semicolon_char); def do_package_proc(self): word = self.getNextword(); self.symbol.update_namespace(word) self.getSpecialChar(self.semicolon_char); token_set = { 'message':do_message_proc ,'enum':do_enum_proc ,'import':do_import_proc ,'package':do_package_proc } def do_expression(self): while 1: current_index = self.backup(); token = self.getNextword(); if(token == None): break; if(token in self.token_set): proc = self.token_set[token]; proc(self); else: self.restore(current_index) break;
def __init__(self,sBuf,symbol): self.Buf = sBuf; self.index = 0; self.count = len(self.Buf.Data) self.symbol = symbol;
3 symbol--定义对象类型以及层次
# -*- coding: UTF-8 -*- # pb_symbol.py
import os import string import pb_typecollection class PBEntity(object): def __init__(self,entName,rtname): self.entName = entName; self.orgName = entName self.rtname = rtname def outputDebug(self): pass; def create_impl(self,entity_indent,top_ns): batch_list = list(); return batch_list; def mem_include(self,entName): return False; class PBMessageMember(object): def __init__(self,option,memType,memText): self.option = option; self.memType = memType; self.memText = memText; def outputDebug(self): print(self.option,self.memType,self.memText) @property def mem_option(self): return self.option @property def mem_type(self): return self.memType; @property def mem_text(self): return self.memText class PBMessage(PBEntity): def __init__(self,entName): PBEntity.__init__(self,entName, entName ); self.members = [] self.enableSymbol = 0; self.rt_ns = ''; self.tpDict = None @property def Members(self): return self.members def attach_tp_dict(self,tpDict): self.tpDict = tpDict; def append_Member(self,option,memType,memText): msgMem = PBMessageMember(option,memType,memText) self.members.append(msgMem) def enable_Symbol(self,enable): self.enableSymbol = enable; def outputDebug(self,ns): print(ns,'message',self.entName); for entMsg in self.members: entMsg.outputDebug(); print(''); def attach_tp_dict(self,tpDict): self.tpDict = tpDict; def set_rt_ns(self,rt_entity_full_path): self.rt_ns = rt_entity_full_path def mem_include(self,entName): for entMsg in self.members: if(entName == entMsg.memType): return True; return False; def detect_request(self): if(self.members.count > 0 ): return True; return False; class PBEnumMember(object): def __init__(self,memText,memValue): self.memText = memText; self.memValue = memValue; def outputDebug(self): print(self.memText,self.memValue) class PBEnum( PBEntity): def __init__(self,entName): PBEntity.__init__(self,entName,entName); self.members = [] def append_Member(self,memText,memValue): msgMem = PBEnumMember(memText,memValue) self.members.append(msgMem) def outputDebug(self,ns): print(ns,'enum',self.entName); for entEnum in self.members: entEnum.outputDebug(); print(''); class Symbol(object): def __init__(self,tpDict,fullpath,rooted): self.namespace = '' self.tpDict = tpDict self.rooted = rooted self.entity_full_path = fullpath self.rt_entity_full_path = fullpath self.entitylist = [] self.containerlist = [] def __del__(self): pass; def update_namespace(self,namespace): self.namespace = namespace; if(self.rooted == False): if(self.entity_full_path == ''): self.entity_full_path = namespace self.rt_entity_full_path = namespace else: self.entity_full_path = '%s_%s' %(self.entity_full_path,namespace) self.rt_entity_full_path = '%s_%s' %(self.entity_full_path,namespace) def append_type_dict(self,entity,isMsg): if(isMsg == True): if(self.entity_full_path == ''): self.tpDict.insert_type(entity.entName ,entity.rtname ,entity ,'') else: self.tpDict.insert_type(entity.entName ,'%s::%s' % (self.rt_entity_full_path, entity.rtname) ,entity ,'') else: if(self.entity_full_path == ''): self.tpDict.insert_type(entity.entName ,entity.rtname ,entity ,entity.rtname) else: self.tpDict.insert_type(entity.entName ,'%s::%s' % (self.rt_entity_full_path, entity.rtname) ,entity ,'%s::%s' % (self.entity_full_path, entity.rtname)) def append_message(self,msg): self.entitylist.append(msg) self.containerlist.append(msg) msg.attach_tp_dict(self.tpDict); if(self.rt_entity_full_path == ''): msg.set_rt_ns(self.rt_entity_full_path) else: msg.set_rt_ns(self.rt_entity_full_path + '_') self.append_type_dict(msg,True) def append_enum(self,enum): self.entitylist.append(enum) self.append_type_dict(enum,False) def append_symbol(self,symbol): self.entitylist.append(symbol) self.containerlist.append(symbol) def outputDebug(self,ns): for entity in self.entitylist: entity.outputDebug(ns +'::'+self.namespace); def query_entitylist(self): return self.entitylist; def query_containerlist(self): return self.containerlist; def query_pb_ns(self): return self.namespace; def mem_include(self,entName): for entity in self.entitylist: if(entity.mem_include(entName) == True): return True; return False; class PBProxy(object): def __init__(self,entity): self.entity = entity @property def enableSymbol(self): return self.entity.enableSymbol def mem_include(self,entName): return self.entity.mem_include(entName) def create_impl(self,entity_indent,top_ns): return self.entity.create_impl(entity_indent,top_ns) @property def entName(self): return self.entity.entName; @property def rtname(self): return self.entity.rtname; @property def orgName(self): return self.entity.orgName; @property def members(self): return self.entity.members; @property def rt_ns(self): return self.entity.rt_ns; @property def namespace(self): return self.entity.namespace; @property def rooted(self): return self.entity.rooted; @property def entity_full_path(self): return self.entity.entity_full_path; @property def rt_entity_full_path(self): return self.entity.rt_entity_full_path; @property def entitylist(self): return self.entity.entitylist @property def containerlist(self): return self.entity.containerlist @property def tpDict(self): return self.entity.tpDict; def detect_request(self): return self.entity.detect_request() @property def Members(self): return self.entity.members @property def mem_option(self): return self.entity.mem_option @property def mem_type(self): return self.entity.mem_type; @property def mem_text(self): return self.entity.mem_text
4 typecollection
# -*- coding: UTF-8 -*-
# pb_typecollection.py
import os
import pb_symbol
class typeDict(object):
op_req_desc = 'required'
op_opt_desc = 'optional'
op_rep_desc = 'repeated'
def __init__(self):
self.collection = dict()
self.insert_type('int32','__int32',pb_symbol.PBEntity('int32','int32'),'')
self.insert_type('int64','__int64',pb_symbol.PBEntity('int64','int64'),'')
self.insert_type('uint32','unsigned int',pb_symbol.PBEntity('uint32','uint32'),'')
self.insert_type('bool','bool',pb_symbol.PBEntity('bool','bool'),'')
self.insert_type('float','float',pb_symbol.PBEntity('float','float'),'')
self.insert_type('double','double',pb_symbol.PBEntity('double','double'),'')
self.insert_type('string','const char*',pb_symbol.PBEntity('string','string'),'')
self.insert_type('bytes','const char*',pb_symbol.PBEntity('bytes','bytes'),'')
def insert_type(self, entName, rtType,entity,orgType):
self.collection[entName] = (rtType,entity,orgType);
def output_debug(self):
print('type collection')
for item in self.collection.items():
print(item);
5 测试脚本
# -*- coding: UTF-8 -*-
import pb_symbol
import pb_expression
import pb_typecollection
if __name__ == '__main__':
pb_file = 'google_tutorial.proto'
sBuf = pb_expression.StringBuffer(pb_file);
tpDict = pb_typecollection.typeDict()
symbol = pb_symbol.Symbol(tpDict,'',True);
try:
sBuf.OpenFile();
exp = pb_expression.Expression(sBuf,symbol);
exp.do_expression();
symbol.outputDebug('');
tpDict.output_debug();
except Exception as exc:
print("%s",exc);
print("done");
6 输出
命名空间：::tutorial::Person
类型名称：PhoneType
('::tutorial::Person', 'enum', 'PhoneType')
('MOBILE', '0')
('HOME', '1')
('WORK', '2')
('::tutorial::Person', 'message', 'PhoneNumber')
('required', 'string', 'number')
('optional', 'PhoneType', 'type')
('::tutorial', 'message', 'Person')
('required', 'string', 'name')
('required', 'int32', 'id')
('optional', 'string', 'email')
('repeated', 'PhoneNumber', 'phone')
('::tutorial', 'message', 'AddressBook')
('repeated', 'Person', 'person')
type collection
('PhoneNumber', ('Person::PhoneNumber', <pb_symbol.PBMessage object at 0x02B9DED0>, ''))
('int32', ('__int32', <pb_symbol.PBEntity object at 0x02BE3F70>, ''))
('string', ('const char*', <pb_symbol.PBEntity object at 0x02BEE0F0>, ''))
('double', ('double', <pb_symbol.PBEntity object at 0x02BEE0B0>, ''))
('float', ('float', <pb_symbol.PBEntity object at 0x02BEE070>, ''))
('bytes', ('const char*', <pb_symbol.PBEntity object at 0x02BEE130>, ''))
('Person', ('Person', <pb_symbol.PBMessage object at 0x02BEE210>, ''))
('bool', ('bool', <pb_symbol.PBEntity object at 0x02BEE050>, ''))
('PhoneType', ('Person::PhoneType', <pb_symbol.PBEnum object at 0x02BEE450>, 'Person::PhoneType'))
('int64', ('__int64', <pb_symbol.PBEntity object at 0x02BE3FB0>, ''))
('uint32', ('unsigned int', <pb_symbol.PBEntity object at 0x02BE3FF0>, ''))
('AddressBook', ('AddressBook', <pb_symbol.PBMessage object at 0x02BEE7B0>, ''))
参考
protobuf的git地址：https://github.com/google/protobuf
