自动化测试平台开发（九）：接口测试 - YAPI数据同步
前面两篇记录了后端接口实现、前端页面交互实现的一个示例，后面需要做的是把需求的其他后端接口和前端功能实现，期间踩了许多的坑后，基本实现了核心功能，并demo。
本篇介绍 YAPI数据同步的实现。
1. 原理
2. 源码
mongo操作
1 #!/usr/bin/python 2 # -*- coding:utf-8 _*- 3 """ 4 @author:TXU 5 @file:mongo.py 6 @time:2021/11/11 7 @email:tao.xu2008@outlook.com 8 @description: 9 """ 10 from pymongo import MongoClient 11 from pymongo.errors import ConnectionFailure 12 import unittest 13 14 from loguru import logger as default_logger 15 from common.singleton import Singleton 16 17 18 class MongoDBOperation(object, metaclass=Singleton): 19 """MongoDB数据库操作""" 20 21 def __init__(self, db_host, db_port, db_user, db_password, db_name, 22 mechanism='SCRAM-SHA-1', show=False, logger=default_logger): 23 self.db_host = db_host 24 self.db_port = db_port 25 self.db_user = db_user 26 self.db_password = db_password 27 self.db_name = db_name 28 self.mechanism = mechanism 29 self.show = show 30 self.logger = logger 31 32 self.conn = None 33 self.database = None 34 self.connect() 35 36 def connect(self): 37 """ 38 connect to MangoDB 39 :return: 40 """ 41 uri = "mongodb://%s:%s@%s:%s" % (self.db_user, self.db_password, self.db_host, self.db_port) 42 self.logger.info("连接 {} ...".format(uri)) 43 self.conn = MongoClient(uri, authSource='admin', authMechanism=self.mechanism) 44 try: 45 self.conn.admin.command('ping') 46 self.logger.info("MangoDB server available") 47 except ConnectionFailure as e: 48 self.logger.error("MangoDB server not available") 49 raise e 50 51 try: 52 self.database = self.conn[self.db_name] 53 self.logger.info("数据库: %s 连接成功！" % self.db_name) 54 except Exception as e: 55 raise e 56 57 def close(self): 58 self.logger.info("断开MangoDB数据库的连接...") 59 self.conn.close() 60 61 def read_table(self, table_name, query_filter=None, projection=None, limit=0): 62 """ 63 读取目标表中数据 64 :param table_name: 表名称 65 :param query_filter: 查询条件 {} 66 :param projection: 要读取字段清单 [] / {}, {"_id": False} 67 :param limit: maximum number of results to return. A limit of 0 (the default) is equivalent to setting no limit 68 :return: 69 """ 70 self.logger.info("读取目标表:{}，query_filter:{}".format(table_name, query_filter)) 71 export_table = [] 72 try: 73 table = self.database[table_name] 74 for item in table.find(query_filter, projection, limit=limit): 75 export_table.append(item) 76 self.logger.info("MangoDB表：%s读取完毕，共读取%s行数据！" % (table_name, len(export_table))) 77 except Exception as e: 78 self.logger.error("查询失败或者数据库中无此表！数据表:{}，query_filter:{}".format(table_name, query_filter)) 79 raise e 80 return export_table
获取yapi info
1 #!/usr/bin/python 2 # -*- coding:utf-8 _*- 3 """ 4 @author:TXU 5 @file:get_api_from_mongo.py 6 @time:2021/11/12 7 @email:tao.xu2008@outlook.com 8 @description: 9 """ 10 import json 11 12 # from apps.api_test import logger 13 from loguru import logger 14 from common.mongodb import MongoDBOperation 15 16 17 MONGODB_HOST = '192.168.0.1' 18 MONGODB_PORT = 7211 19 MONGODB_USER = 'user' 20 MONGODB_PWD = 'password' 21 MONGODB_DATABASE = 'yapi' 22 23 24 class YapiMongoDB(MongoDBOperation): 25 """从mongodb数据库获取yapi数据""" 26 def __init__(self, 27 db_host=MONGODB_HOST, 28 db_port=MONGODB_PORT, 29 db_user=MONGODB_USER, 30 db_password=MONGODB_PWD, 31 db_name=MONGODB_DATABASE): 32 super(YapiMongoDB, self).__init__(db_host, db_port, db_user, db_password, db_name, logger=logger) 33 pass 34 35 def get_projects_by_api_full_path(self, api_full_path): 36 # 根据full path查找项目: fullpath.startswith(project.basepath) 37 match_projects = [] 38 projects = self.read_table('project') 39 if len(projects) == 0: 40 raise Exception("YAPI中没找到project") 41 for p in projects: 42 if api_full_path.startswith(p.get('basepath')): 43 match_projects.append(p) 44 return match_projects 45 46 def get_api_by_method_fullpath(self, method, fullpath): 47 # 根据full path查找项目: 项目.basepath in fullpath 48 match_projects = self.get_projects_by_api_full_path(fullpath) 49 # 排序：basepath长度从大到小：优先检索basepath不为空的项目 50 for p in sorted(match_projects, key=lambda x: x.get('basepath'), reverse=True): 51 basepath = p.get('basepath') 52 p_id = p.get('_id') 53 path = fullpath.replace(basepath, '') 54 query_filter = {'project_id': p_id, 'method': method.upper(), 'path': path} 55 apis = self.read_table('interface', query_filter) 56 if len(apis) == 0: 57 continue 58 return apis[0] 59 return None 60 61 def get_api_by_id(self, api_id): 62 query_filter = {'_id': api_id} 63 apis = self.read_table('interface', query_filter) 64 if len(apis) == 0: 65 raise Exception("YAPI中没找到对应接口, {}".format(query_filter)) 66 return apis[0] 67 68 def get_project_by_id(self, project_id): 69 query_filter = {'_id': project_id} 70 projects = self.read_table('project', query_filter) 71 if len(projects) == 0: 72 raise Exception("YAPI中没找到对应project:{}".format(query_filter)) 73 return projects[0] 74 75 def get_api_cat_by_id(self, cat_id): 76 query_filter = {'_id': cat_id} 77 cats = self.read_table('interface_cat', query_filter) 78 if len(cats) == 0: 79 raise Exception("YAPI中没找到对应接口分类:{}".format(query_filter)) 80 return cats[0] 81 82 def pop_data_id(self, data_list): 83 """ 84 去掉data中ObjectId类型字段，如 '_id': ObjectId('607d377f9f23fd3457e2f1ff') 85 req_headers | req_query | yapi_req_body_form 86 :param data_list: 87 :return: 88 """ 89 for data in data_list: 90 data.pop('_id') 91 return data_list 92 93 def parse_api(self, api): 94 """ 95 解析yapi原始数据 96 :param api: 97 :return: 98 """ 99 api_name = api.get("title") 100 api_path = api.get("path") 101 if api_path.startswith('/internal'): 102 # logger.warning('{}->{}(内部接口，跳过!!！)'.format(api_name, api_path)) 103 raise Exception('{}->{}(内部接口，跳过!!！)'.format(api_name, api_path)) 104 api_project_id = api.get("project_id"), 105 api_project = self.get_project_by_id(api_project_id[0]) 106 prj_basepath = api_project.get("basepath") 107 req_body_other_src = api.get('req_body_other', '{}') 108 res_body_src = api.get('res_body', '{}') 109 try: 110 req_body_other = json.loads(req_body_other_src) 111 except Exception as e: 112 req_body_other = {"JSONDecodeError": str(e), "body": req_body_other_src} 113 try: 114 res_body = json.loads(api.get('res_body', '{}')) 115 except Exception as e: 116 res_body = {"JSONDecodeError": str(e), "body": res_body_src} 117 118 api_info = { 119 "name": api_name, 120 "description": api.get("desc"), 121 "yapi_id": api.get("_id"), 122 "yapi_project_id": api.get("project_id"), # 本地接口表无此字段 123 "yapi_cat_id": api.get("catid"), # 本地接口表无此字段 124 "method": api.get("method"), 125 "path": prj_basepath + api_path, 126 "yapi_req_headers": self.pop_data_id(api.get('req_headers', [])), 127 "yapi_req_params": self.pop_data_id(api.get('req_params', [])), 128 "yapi_req_query": self.pop_data_id(api.get('req_query', [])), 129 "yapi_req_body_form": self.pop_data_id(api.get('req_body_form', [])), 130 "yapi_req_body_other": req_body_other, 131 "yapi_res_body": res_body 132 } 133 return api_info 134 135 def get_api_info_by_id(self, yapi_id): 136 """ 137 根据yapi id获取yapi接口详情，并解析 138 :param yapi_id: 139 :return: 140 """ 141 yapi = self.get_api_by_id(yapi_id) 142 return self.parse_api(yapi) 143 144 def get_api_info_by_method_path(self, method, api_full_path): 145 """ 146 根据api_full_path获取yapi接口详情，并解析 147 :param method: 请求方法 148 :param api_full_path: 请求path，全路径 149 :return: 150 """ 151 yapi = self.get_api_by_method_fullpath(method, api_full_path) 152 if not yapi: 153 raise Exception("YAPI中没找到对应接口, {} {}".format(method, api_full_path)) 154 return self.parse_api(yapi) 155 156 157 if __name__ == '__main__': 158 yapi_db = YapiMongoDB() 159 yapi_info = yapi_db.get_api_by_method_fullpath('GET', '/xxxx/setting/get_setting') 160 print(yapi_info)
view:
1 #!/usr/bin/python 2 # -*- coding:utf-8 _*- 3 """ 4 @author:TXU 5 @file:update.py 6 @time:2021/11/11 7 @email:tao.xu2008@outlook.com 8 @description: 9 """ 10 import traceback 11 12 from rest_framework.views import APIView 13 from apps.api_test import logger 14 from apps.api_test.view_set.base import JsonResponse 15 from apps.api_test.models import YApiEvent 16 from apps.api_test.yapi.yapi_sync import APICaseTemplate, YAPIEventDataSync 17 18 19 # 用例模板更新：读取接口YAPI定义数据，生成并更新接口请求用例模板 20 class APICaseTemplateUpdateView(APIView): 21 """读取接口YAPI定义数据，生成并更新接口请求用例模板""" 22 def post(self, request, *args, **kwargs): 23 api_list = request.data 24 api_temp = APICaseTemplate() 25 if 'all' in api_list: 26 res_list = api_temp.update_all() 27 else: 28 res_list = api_temp.bulk_update_template(api_list) 29 response = {"msg": "用例模板更新完成！", "data": res_list} 30 return JsonResponse(response, status=200) 31 32 33 # yapi事件监听：YAPI web hook事件监听，接受事件并插入YAPIEvent表 34 class YAPIEventMonitorView(APIView): 35 def post(self, request, *args, **kwargs): 36 """ 37 YAPI接口变更事件监听处理 38 1. 客户端更新接口成功后触发 39 {'_id': 365459, 'YApiEvent': 'yapi_interface_update'} 40 2. 客户端新增接口成功后触发 41 TODO 42 3. 客户端删除接口成功后触发 43 TODO 44 :param request: 45 """ 46 data = request.data 47 print('data:{}'.format(data)) 48 yapi_id = data.get("_id", 0) 49 event = data.get("YApiEvent", "null") 50 exist_events = YApiEvent.objects.filter(yapi_id__exact=yapi_id, event__exact=event) 51 if exist_events.count() > 0: 52 response = { 53 "msg": "yapi变更事件已存在，忽略", 54 "data": data 55 } 56 else: 57 YApiEvent.objects.create( 58 **{ 59 'yapi_id': yapi_id, 60 'event': event, 61 'content': data, 62 } 63 ) 64 response = { 65 "msg": "yapi变更事件添加成功", 66 "data": data 67 } 68 69 return JsonResponse(response, status=200) 70 71 72 # yapi数据同步：YAPI变更事件->同步数据到本地接口 73 class YAPIEventDataSyncView(APIView): 74 """YAPI变更事件->同步数据到本地接口""" 75 76 def post(self, request, *args, **kwargs): 77 """ 78 {'list':'all'} -- data_sync() 79 {'list':[]} -- data_sync_single(yapi_event) 80 :param request: 81 :param args: 82 :param kwargs: 83 :return: 84 """ 85 yapi_event_list = request.data 86 yapi_data_sync = YAPIEventDataSync() 87 if 'all' in yapi_event_list: 88 res_list = yapi_data_sync.data_sync_all() 89 else: 90 res_list = yapi_data_sync.data_sync_bulk(yapi_event_list) 91 response = {"msg": "更新处理完成！", "data": res_list} 92 93 return JsonResponse(response, status=200) 94 95 96 # 接口拉取同步：本地接口主动拉取YAPI数据同步 97 class APIDataSyncView(APIView): 98 """本地接口主动拉取YAPI数据同步""" 99 100 def post(self, request, *args, **kwargs): 101 api_list = request.data 102 v_yapi_events = [] 103 yapi_data_sync = YAPIEventDataSync() 104 for api in api_list: 105 logger.info("更新接口：{} -> {}".format(api.get('id'), api.get('name'))) 106 yapi_id = api.get('yapi_id', '') 107 # 构建类YAPIEvent字典，调用YAPIEventDataSync同步数据 108 v_yapi_events.append({ 109 'id': 0, 110 'yapi_id': yapi_id, 111 'event': 'pull_yapi_info', 112 'api_id': api.get('id'), 113 'method': api.get('method'), 114 'path': api.get('path'), 115 }) 116 res_list = yapi_data_sync.data_sync_bulk(v_yapi_events) 117 response = {"msg": "更新处理完成！", "data": res_list} 118 return JsonResponse(response, status=200) 119 120 121 if __name__ == '__main__': 122 pass
url路由：
1 # YAPI事件监听 2 url(r'api/yapi_event/add', YAPIEventMonitorView.as_view()), 3 # YAPI事件数据同步到本地接口 4 url(r'api/yapi_event/data_sync', YAPIEventDataSyncView.as_view()), 5 # 本地接口拉取YAPI数据同步 6 url(r'api/data_sync', APIDataSyncView.as_view()), 7 # 本地接口用例模板更新 8 url(r'api/update/case_template', APICaseTemplateUpdateView.as_view()), 9 # 接口update_status同步到接口变更历史表 10 url(r'api/sync/update_history/update_status', ApiSyncUpdateStatusToHistoryViewSet.as_view()),
-------- THE END --------
