语料处理 合集

1.中文语料常常遇到编码问题，将任意字符集文件转为utf-8编码

import chardet 
import codecs
from django.utils.encoding import smart_text
 
def check_file_charset(file):                 #查看file文件的编码 
    with open(file, 'rb') as f:
        return chardet.detect(f.read())

def Convert_file_character(File_path):
    f_type = check_file_charset(File_path)
    print (File_path,"字符集为：",f_type['encoding'])       
    try:
        if f_type and 'encoding' in f_type.keys() and f_type['encoding'] != 'utf-8':
            with codecs.open(File_path, 'rb', f_type['encoding'],errors='ignore') as f:
                content = smart_text(f.read())
            with codecs.open(File_path, 'wb', 'utf-8') as f:
                f.write(content)
            print ("字符集转换成功")
        else:
            print("字符集为 utf-8，不需要进行转换")
    except Exception as ERR:
        print("字符集转换失败")
        print (ERR)

corpus_path = './unlabel'
raw_train_files = [corpus_path + os.sep + file_name for file_name in os.listdir(corpus_path)]
for raw_train_file in raw_train_files:
    Convert_file_character(raw_train_file)

参考：https://blog.csdn.net/qq_35751770/article/details/103664496

2.将unlabel文件夹中的所有.txt文件合并，每个文件之间空一行 

先调用上面的代码转换编码

def combine(corpus_path, outpath):
    output = open(outpath, 'a', encoding='utf-8')
    
    raw_train_files = [corpus_path + os.sep + file_name for file_name in os.listdir(corpus_path)]
    for raw_train_file in raw_train_files:
        
        f_type = check_file_charset(raw_train_file)                    #查看文件的编码 
        print (raw_train_file,"字符集为：",f_type['encoding'])
        with open(raw_train_file, 'r+', encoding='utf-8') as f:
            context = f.readlines()
    
        for x in context:
            output.write(x)
        output.write('\n')

combine('./unlabel', 'all_unlabel.txt')

3.随机抽取.txt文件中的60%，20%，5%

def part(filename, outpath, ratio): 
    output = open(outpath, 'w+', encoding='utf-8')
    context = []
    with open(filename, 'r+', encoding='utf-8') as f:
        context.extend(f.readlines())
    
    length = len(context)
    random_order = list(range(length))
    np.random.shuffle(random_order)
    
    batch_size = int(length*ratio)
    print(batch_size)
    for x in context[:batch_size]:
        output.write(x)
        
ratio1, ratio2, ratio3 = 0.6, 0.2, 0.05
part('training/law_train.txt', 'training/law_train1.txt', ratio1)    
part('training/law_train.txt', 'training/law_train2.txt', ratio2)           
part('training/law_train.txt', 'training/law_train3.txt', ratio3) 

4.将已经分好词的文件去掉空格（正则），恢复成文件原来的样子

def deal_data(filename, outpath): 
    output = open(outpath, 'w+', encoding='utf-8')
    
    with open(filename, 'r+', encoding='utf-8') as f:
        context = f.readlines()
        for data in context:                 #data为某一行数据             
            x = re.sub('\s+', '', data).strip()
            output.write(x)
            

deal_data('evaluate/law/Law_contract_test.txt', 'evaluate/gold/Law_contract_test.txt')
deal_data('evaluate/law/Law_marriage_test.txt', 'evaluate/gold/Law_marriage_test.txt')            
deal_data('evaluate/law/Law_mixed_test.txt', 'evaluate/gold/Law_mixed_test.txt')

5.读取excel文件转换成.json文件

#coding=utf-8
import xlrd        #对excel文件内容读取
import xlwt        #对excel文件内容写入 
import json
"""
打开excel文件 处理成json文件 {text：，label：}
data.xls变成train.json、val.json、test.json
"""

def deal_data(filename,outpath):              #filename为xlsx文件路径 outputfile为json文件路径
    wb = xlrd.open_workbook(filename)         #打开excel文件读取数据 
    data_file=["train","test","val"]

    for excel_name in data_file:
        output_file = outpath + excel_name+".json"              #命名处理之后的json文件名 
        output = open(output_file, "w", encoding="utf-8")       #写入 

        excel = wb.sheet_by_name(excel_name)    #根据sheet名称获取sheet内容
        rows_n = excel.nrows                    #同时获取sheet总行数
        for i in range(rows_n):                                 #分别获取每行的第0、1、2列 
            data_dic = {}
            data_dic["filepath"] = excel.cell_value(i , 0)            
            data_dic["text"] = excel.cell_value(i , 1).strip()
            data_dic["label"] = tuple(excel.cell_value(i , 2).split())

            output.write(json.dumps(data_dic) + "\n")           #写入json文件 
        output.close()

deal_data("data01.xls","corpus/class/origin_corpus/")