python异步字符串查找，asyncio和marisa_trie

 

自然语言处理当中经常需要字符串的查找操作，比如通过查找返回字串在文本当中的位置，比如通过匹配实现的ner

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58

import pandas as pd import asyncio   # data = pd.read_csv("guba_fc_result_20230413.csv")   data = pd.read_csv("guba_all_post_20230413.csv") filename = "cate_group.txt"   def read_list_from_file(filename):     with open(filename, 'r') as f:         lst = [line.strip() for line in f]     return lst   cate_group = read_list_from_file(filename)     import marisa_trie # 构建trie trie = marisa_trie.Trie(cate_group)    # 匹配长字符串 # long_string = '宁德时代是做锂电池的'    # results = [] # for i in range(len(long_string)): #     matches = trie.prefixes(long_string[i:]) #     # 输出匹配结果 #     if matches: #         for matche in matches: #             results.append(matche)    # print(results)     async def match_text(long_string):     long_string = str(long_string)     results = []     for i in range(len(long_string)):         matches = trie.prefixes(long_string[i:])         # 输出匹配结果         if matches:             for matche in matches:                 results.append(matche)     return results    async def main():     tasks = []     for i in data["text"]:         tasks.append(asyncio.create_task(match_text(i)))     matches_list = await asyncio.gather(*tasks)       data["matches"] = matches_list       print(matches_list)     data.to_csv("guba_all_matches_20230413.csv")    if __name__ == '__main__':     asyncio.run(main())