使用单卡qlora混合精度训练大模型chatGLM2-6b，解决qlora loss变成nan的问题！

 

最近新换了工作，以后的工作内容会和大模型相关，所以先抽空跑了一下chatGLM2-6b的demo，使用Qlora或lora微调模型

今天简单写个文档记录一下，顺便也是一个简单的教程，并且踩了qlora loss变成nan训练不稳定的问题

本教程并没有写lora的原理，需要的话自行查阅

 

1.chatGLM2-6b 模型我已经从huggingface 下载到服务器，因为我的服务器不能直接连接huggingface 下载

我是放到了文件夹下 /data/tmp/chatGLM2_6b_pretrain,包含模型文件和一些配置文件，直接在huggingface下载就好

 

2.打印模型结构

from transformers import AutoModel

model_name = "/data/tmp/chatGLM2_6b_pretrain"
model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
print(model)

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29

ChatGLMForConditionalGeneration(   (transformer): ChatGLMModel(     (embedding): Embedding(       (word_embeddings): Embedding(65024, 4096)     )     (rotary_pos_emb): RotaryEmbedding()     (encoder): GLMTransformer(       (layers): ModuleList(         (0-27): 28 x GLMBlock(           (input_layernorm): RMSNorm()           (self_attention): SelfAttention(             (query_key_value): Linear(in_features=4096, out_features=4608, bias=True)             (core_attention): CoreAttention(               (attention_dropout): Dropout(p=0.0, inplace=False)             )             (dense): Linear(in_features=4096, out_features=4096, bias=False)           )           (post_attention_layernorm): RMSNorm()           (mlp): MLP(             (dense_h_to_4h): Linear(in_features=4096, out_features=27392, bias=False)             (dense_4h_to_h): Linear(in_features=13696, out_features=4096, bias=False)           )         )       )       (final_layernorm): RMSNorm()     )     (output_layer): Linear(in_features=4096, out_features=65024, bias=False)   ) )

?

1	query_key_value 这个矩阵不是三个方阵拼接到一起，应该是Wq 4096*4096  Wk 4096*256 Wv 4096*256 使用的 group-attention

　3.打印添加lora后的模型结构

from transformers import AutoTokenizer, AutoModel, AutoConfig
from peft import LoraConfig, get_peft_model, TaskType

model_name = "/data/tmp/chatGLM2_6b_pretrain"
model = AutoModel.from_pretrained(model_name, trust_remote_code=True)

config = LoraConfig(
    peft_type="LORA",
    task_type=TaskType.CAUSAL_LM,
    inference_mode=False,
    r=8,
    lora_alpha=16,
    lora_dropout=0.1,
    fan_in_fan_out=False,
    bias='lora_only',
    target_modules=["query_key_value"]
)

model = get_peft_model(model, config)
print(model)

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46

PeftModelForCausalLM(   (base_model): LoraModel(     (model): ChatGLMForConditionalGeneration(       (transformer): ChatGLMModel(         (embedding): Embedding(           (word_embeddings): Embedding(65024, 4096)         )         (rotary_pos_emb): RotaryEmbedding()         (encoder): GLMTransformer(           (layers): ModuleList(             (0-27): 28 x GLMBlock(               (input_layernorm): RMSNorm()               (self_attention): SelfAttention(                 (query_key_value): Linear(                   in_features=4096, out_features=4608, bias=True                   (lora_dropout): ModuleDict(                     (default): Dropout(p=0.1, inplace=False)                   )                   (lora_A): ModuleDict(                     (default): Linear(in_features=4096, out_features=8, bias=False)                   )                   (lora_B): ModuleDict(                     (default): Linear(in_features=8, out_features=4608, bias=False)                   )                   (lora_embedding_A): ParameterDict()                   (lora_embedding_B): ParameterDict()                 )                 (core_attention): CoreAttention(                   (attention_dropout): Dropout(p=0.0, inplace=False)                 )                 (dense): Linear(in_features=4096, out_features=4096, bias=False)               )               (post_attention_layernorm): RMSNorm()               (mlp): MLP(                 (dense_h_to_4h): Linear(in_features=4096, out_features=27392, bias=False)                 (dense_4h_to_h): Linear(in_features=13696, out_features=4096, bias=False)               )             )           )           (final_layernorm): RMSNorm()         )         (output_layer): Linear(in_features=4096, out_features=65024, bias=False)       )     )   ) )

　　会发现 在query_key_value 矩阵下 多了两个全连接层，lora_A 和 lora_B ，这两个全连接层就是要训练的

 

4.准备数据集，我们使用的firefly数据集,可以自行去huggingface下载jsonl格式，需要提前划分好训练集和测试集 qa_dataset.py

# -*- coding: utf-8 -*-
from torch.utils.data import Dataset
import torch
import json
import numpy as np


class QADataset(Dataset):
    def __init__(self, data_path, tokenizer, max_source_length, max_target_length) -> None:
        super().__init__()
        self.tokenizer = tokenizer
        self.max_source_length = max_source_length
        self.max_target_length = max_target_length
        self.max_seq_length = self.max_source_length + self.max_target_length

        self.data = []
        with open(data_path, "r", encoding='utf-8') as f:
            for line in f:
                if not line or line == "":
                    continue
                json_line = json.loads(line)
                # {'kind': 'NLI', 'input': '自然语言推理：\n前提：家里人心甘情愿地养他,还有几家想让他做女婿的\n假设：他是被家里人收养的孤儿', 'target': '中立'}
                kind = json_line["kind"]
                input = json_line["input"]
                target = json_line["target"]
                self.data.append({
                    "question": input,
                    "answer": "--**"+kind+"**--\n"+target
                })
        print("data load ， size：", len(self.data))
    def preprocess(self, question, answer):
        prompt = self.tokenizer.build_prompt(question, None)

        a_ids = self.tokenizer.encode(text=prompt, add_special_tokens=True, truncation=True,
                                      max_length=self.max_source_length)

        b_ids = self.tokenizer.encode(text=answer, add_special_tokens=False, truncation=True,
                                      max_length=self.max_target_length-1) #因为会补充eos_token

        context_length = len(a_ids)
        input_ids = a_ids + b_ids + [self.tokenizer.eos_token_id]
        labels = [self.tokenizer.pad_token_id] * context_length + b_ids + [self.tokenizer.eos_token_id]

        pad_len = self.max_seq_length - len(input_ids)
        input_ids = input_ids + [self.tokenizer.pad_token_id] * pad_len
        labels = labels + [self.tokenizer.pad_token_id] * pad_len
        labels = [(l if l != self.tokenizer.pad_token_id else -100) for l in labels]
        return input_ids, labels

    def __getitem__(self, index):
        item_data = self.data[index]

        input_ids, labels = self.preprocess(**item_data)

        return {
            "input_ids": torch.LongTensor(np.array(input_ids)),
            "labels": torch.LongTensor(np.array(labels))
        }

    def __len__(self):
        return len(self.data)

if __name__ == "__main__":
    with open("/data/tmp/firefly_data/firefly-train-1.1M.jsonl", "r", encoding='utf-8') as f_read,open("/data/tmp/firefly_data/firefly_train80000.jsonl","w",encoding='utf-8') as f_trainx, open("/data/tmp/firefly_data/firefly_train.jsonl","w",encoding='utf-8') as f_train, open("/data/tmp/firefly_data/firefly_test.jsonl","w",encoding='utf-8') as f_test:
        lines = f_read.readlines()

        f_test.writelines(lines[:1000])
        f_train.writelines(lines[1000:])
        f_trainx.writelines(lines[1000:81000])

 

5.训练lora，使用半精度，占用显存很大，batch_size只能为1，显存就要占用到30g了，而且训练很久，为了解决这个显存占用大的问题，后面又尝试了qlora

train_lora.py

# -*- coding: utf-8 -*-
import pandas as pd
from torch.utils.data import DataLoader
from transformers import AutoTokenizer, AutoModel
from qa_dataset import QADataset
from peft import LoraConfig, get_peft_model, TaskType
from tqdm import tqdm
import torch
import os, time, sys
import numpy as np


def train(epoch, model, device, loader, optimizer, gradient_accumulation_steps,model_output_dir):
    model.train()
    time1 = time.time()
    losses = []
    train_bar = tqdm(loader,total=len(loader))
    for index, data in enumerate(train_bar):
        input_ids = data['input_ids'].to(device, dtype=torch.long)
        labels = data['labels'].to(device, dtype=torch.long)

        outputs = model(
            input_ids=input_ids,
            labels=labels,
        )
        loss = outputs.loss
        # 反向传播，计算当前梯度
        loss.backward()
        losses.append(loss.item())
        # 梯度累积步数
        if (index % gradient_accumulation_steps == 0 and index != 0) or index == len(loader) - 1:
            # 更新网络参数
            optimizer.step()
            # 清空过往梯度
            optimizer.zero_grad()

        if index % 300 == 0:
            model_save_path = os.path.join(model_output_dir,"index_{}".format(index))
            if os.path.exists(model_save_path):
                pass
            else:
                os.makedirs(model_save_path)
            model.save_pretrained(model_save_path)
        train_bar.set_description("epoch:{} idx:{} loss:{:.6f}".format(epoch,index,np.mean(losses)))



def validate(tokenizer, model, device, loader, max_length):
    model.eval()
    predictions = []
    actuals = []
    with torch.no_grad():
        for _, data in enumerate(tqdm(loader, file=sys.stdout, desc="Validation Data")):
            input_ids = data['input_ids'].to(device, dtype=torch.long)
            labels = data['labels'].to(device, dtype=torch.long)
            generated_ids = model.generate(
                input_ids=input_ids,
                max_length=max_length,
                do_sample=False,
                temperature=0
            )
            preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in
                     generated_ids]
            target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True) for t in labels]
            predictions.extend(preds)
            actuals.extend(target)
    return predictions, actuals


def main():
    model_name = "/data/tmp/chatGLM2_6b_pretrain"
    train_json_path = "/data/tmp/firefly_data/firefly_train20000.jsonl"
    val_json_path = "/data/tmp/firefly_data/firefly_test.jsonl"
    max_source_length = 60
    max_target_length = 360
    epochs = 1
    batch_size = 1
    lr = 1e-4
    lora_rank = 8
    lora_alpha = 32
    gradient_accumulation_steps = 16
    model_output_dir = "output"
    # 设备
    device = torch.device("cuda:0")

    # 加载分词器和模型
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    model = AutoModel.from_pretrained(model_name, trust_remote_code=True)

    # setup peft
    peft_config = LoraConfig(
        task_type=TaskType.CAUSAL_LM,
        inference_mode=False,
        r=lora_rank,
        lora_alpha=lora_alpha,
        lora_dropout=0.1
    )
    model = get_peft_model(model, peft_config)
    model.is_parallelizable = True
    model.model_parallel = True
    model.print_trainable_parameters()
    # 转为半精度
    model = model.half()
    model.float()

    print("Start Load Train Data...")
    train_params = {
        "batch_size": batch_size,
        "shuffle": True,
        "num_workers": 0,
    }
    training_set = QADataset(train_json_path, tokenizer, max_source_length, max_target_length)
    training_loader = DataLoader(training_set, **train_params)
    print("Start Load Validation Data...")
    val_params = {
        "batch_size": batch_size,
        "shuffle": False,
        "num_workers": 0,
    }
    val_set = QADataset(val_json_path, tokenizer, max_source_length, max_target_length)
    val_loader = DataLoader(val_set, **val_params)

    optimizer = torch.optim.AdamW(params=model.parameters(), lr=lr)
    model = model.to(device)
    print("Start Training...")
    for epoch in range(epochs):
        train(epoch, model, device, training_loader, optimizer, gradient_accumulation_steps,model_output_dir)
        # print("Save Model To ", model_output_dir)
        # model.save_pretrained(model_output_dir)
    # 验证
    print("Start Validation...")
    with torch.no_grad():
        predictions, actuals = validate(tokenizer, model, device, val_loader, max_target_length)
        # 验证结果存储
        final_df = pd.DataFrame({"Generated Text": predictions, "Actual Text": actuals})
        val_data_path = os.path.join(model_output_dir, "predictions.csv")
        final_df.to_csv(val_data_path)
        print("Validation Data To ", val_data_path)


if __name__ == '__main__':
    main()

 

 

6.有很多同学手里没有v100 32g显卡，即便batch_size=1 也没办法训练，所以又研究了qlora训练，qlora相比于lora来说就是多了一个模型量化过程，他会把原模型量化得到4-bit NormalFloat，这就会让原模型的显存占用很低，就可以让更多的显存来存放lora部分的参数

但是4-bit NormalFloat 结合手动half半精度的混合精度训练会导致loss很不稳定，可能一跑起来就变成nan了，就连float32结合半精度float16也是不稳定的，所以这块踩了坑，手动使用model.half() loss总会变成nan值，最后使用torch官方的自动混合精度就好了，它会自动的进行混合精度和梯度缩放，不至于产生超过半精度上下限的nan值。

这里说明为什么使用fp16混合精度，而不用float32，实践发现使用混合精度训练可以提速5-6倍的训练时间，在大模型上动不动就要跑很久，时间成本很高。

 

在2w的训练数据下，

qlora半精度需要2个小时就可以训练完成但是不稳定，float32需要11+小时

lora半精度需要不到5个小时，而且会比较稳定

如果这样的话，qlora不能使用半精度训练，否则loss不稳定，使用float32的话就是时间换空间，qlora除了能降低显存就完全失去了优势，还好torch官方的混合精度可以拯救半精度训练的问题

trian_qlora.py

 

# -*- coding: utf-8 -*-
import pandas as pd
from torch.utils.data import DataLoader
from transformers import AutoTokenizer, AutoModel,BitsAndBytesConfig
from qa_dataset import QADataset
from peft import LoraConfig, get_peft_model, TaskType,prepare_model_for_kbit_training
from tqdm import tqdm
import torch
import os, time, sys
from transformers import (
    set_seed,
    HfArgumentParser,
    TrainingArguments,
    AutoModelForCausalLM
)
import bitsandbytes as bnb
from collections import defaultdict
import numpy as np
import os

def verify_model_dtype(model):
    """
    查看模型种各种类型的参数的情况
    """
    dtype2param_num = defaultdict(int)  # 每种数据类型的参数量
    dtype2param_name = defaultdict(list)  # 每种数据类型的参数名称
    dtype2trainable_param_num = defaultdict(int)  # 每种数据类型参与训练的参数量
    dtype2trainable_param_name = defaultdict(list)  # 每种数据类型参与训练的参数名称
    for name, p in model.named_parameters():
        dtype = p.dtype
        dtype2param_num[dtype] += p.numel()
        dtype2param_name[dtype].append(name)
        if p.requires_grad:
            dtype2trainable_param_num[dtype] += p.numel()
            dtype2trainable_param_name[dtype].append(name)
    # 统计全部参数中，各种类型参数分布
    total = 0
    print('verify all params of the model')
    for k, v in dtype2param_num.items():
        total += v
    for k, v in dtype2param_num.items():
        print(k, v, v / total)
    for k, v in dtype2trainable_param_name.items():
        print(k, v)

    print()
    # 统计可训练参数中，各种类型参数分布
    print('verify trainable params the model')
    total_trainable = 0
    for k, v in dtype2trainable_param_num.items():
        total_trainable += v
    for k, v in dtype2trainable_param_num.items():
        print(k, v, v / total_trainable)
    for k, v in dtype2trainable_param_num.items():
        print(k, v)

def find_all_linear_names(model):
    """
    找出所有全连接层，为所有全连接添加adapter
    """
    cls = bnb.nn.Linear4bit
    lora_module_names = set()
    for name, module in model.named_modules():
        if isinstance(module, cls):
            names = name.split('.')
            lora_module_names.add(names[0] if len(names) == 1 else names[-1])

    if 'lm_head' in lora_module_names:  # needed for 16-bit
        lora_module_names.remove('lm_head')
    return list(lora_module_names)

def train(epoch, model, device, loader, optimizer,scaler, gradient_accumulation_steps,model_output_dir):
    model.train()
    time1 = time.time()
    losses = []
    train_bar = tqdm(loader,total=len(loader))
    for index, data in enumerate(train_bar):
        optimizer.zero_grad()
        with torch.autocast(device_type="cuda",dtype=torch.float16):
            input_ids = data['input_ids'].to(device, dtype=torch.long)
            labels = data['labels'].to(device, dtype=torch.long)

            outputs = model(
                input_ids=input_ids,
                labels=labels,
            )
            loss = outputs.loss
            losses.append(loss.item())

            scaler.scale(loss).backward()
            # Unscales the gradients of optimizer's assigned params in-place
            # scaler.unscale_(optimizer)

            # Since the gradients of optimizer's assigned params are unscaled, clips as usual:
            # torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)

            # optimizer's gradients are already unscaled, so scaler.step does not unscale them,
            # although it still skips optimizer.step() if the gradients contain infs or NaNs.
            scaler.step(optimizer)

            # Updates the scale for next iteration.
            scaler.update()

            # # 反向传播，计算当前梯度
            # loss.backward()
            # optimizer.step()
            # 梯度累积步数
            # if (index % gradient_accumulation_steps == 0 and index != 0) or index == len(loader) - 1:
            #     # 更新网络参数
            #     # optimizer.step()
            #     scaler.step(optimizer)
            #     scaler.update()
            #     # 清空过往梯度
            #     optimizer.zero_grad()

        if index % 300 == 0:
            model_save_path = os.path.join(model_output_dir,"index_{}".format(index))
            if os.path.exists(model_save_path):
                pass
            else:
                os.makedirs(model_save_path)
            model.save_pretrained(model_save_path)
        train_bar.set_description("epoch:{} idx:{} loss:{:.6f}".format(epoch,index,np.mean(losses)))
        # 100轮打印一次 loss
        # if index % 100 == 0 or index == len(loader) - 1:
        #     time2 = time.time()
        #     tqdm.write(
        #         f"{index}, epoch: {epoch} -loss: {str(loss)} ; each step's time spent: {(str(float(time2 - time1) / float(index + 0.0001)))}")


def validate(tokenizer, model, device, loader, max_length):
    model.eval()
    predictions = []
    actuals = []
    with torch.no_grad():
        for _, data in enumerate(tqdm(loader, file=sys.stdout, desc="Validation Data")):
            input_ids = data['input_ids'].to(device, dtype=torch.long)
            labels = data['labels'].to(device, dtype=torch.long)
            generated_ids = model.generate(
                input_ids=input_ids,
                max_length=max_length,
                do_sample=False,
                temperature=0
            )
            preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in
                     generated_ids]
            target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True) for t in labels]
            predictions.extend(preds)
            actuals.extend(target)
    return predictions, actuals


def main():
    model_name = "/data/tmp/chatGLM2_6b_pretrain"
    train_json_path = "/data/tmp/firefly_data/firefly_train80000.jsonl"
    val_json_path = "/data/tmp/firefly_data/firefly_test.jsonl"
    max_source_length = 128
    max_target_length = 512
    epochs = 1
    batch_size = 16
    lr = 1e-4
    lora_rank = 32
    lora_alpha = 32
    gradient_accumulation_steps = 16
    model_output_dir = "output"
    # 设备
    device = torch.device("cuda:0")
    lora_dropout = 0.05

    # 加载分词器和模型
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    # model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
    # 加载模型
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        device_map=0,
        load_in_4bit=True,
        torch_dtype=torch.float16,
        trust_remote_code=True,
        quantization_config=BitsAndBytesConfig(
            load_in_4bit=True,
            bnb_4bit_compute_dtype=torch.float16,
            bnb_4bit_use_double_quant=True,
            bnb_4bit_quant_type="nf4",
            llm_int8_threshold=6.0,
            llm_int8_has_fp16_weight=False,
        ),
    )

    # casts all the non int8 modules to full precision (fp32) for stability
    model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True)
    print(f'memory footprint of model: {model.get_memory_footprint()/(1024*1024*1024)} GB')

    # 找到所有需要插入adapter的全连接层
    target_modules = find_all_linear_names(model)
    print("全连接层：",target_modules)
    # 初始化lora配置
    peft_config = LoraConfig(
        r=lora_rank,
        lora_alpha=lora_alpha,
        # target_modules=target_modules,
        target_modules=["query_key_value","dense","dense_h_to_4h","dense_4h_to_h"],
        lora_dropout=lora_dropout,
        bias="none",
        task_type="CAUSAL_LM",
    )

    model = get_peft_model(model, peft_config)


    # model.is_parallelizable = True
    # model.model_parallel = True
    model.print_trainable_parameters()
    # 转为半精度
    # model = model.half() #You shouldn’t call half manually on the model or data.  不使用torch官方的自动混合精度和梯度缩放，手动使用half()半精度会导致模型loss变成nan，使用官方的混合精度需要关闭half()手动半精度，不然会报错
    # model.float()
    model.config.torch_dtype = torch.float32
    # 查看模型种各种类型的参数的情况
    verify_model_dtype(model)

    print(model)

    print("Start Load Train Data...")
    train_params = {
        "batch_size": batch_size,
        "shuffle": True,
        "num_workers": 0,
    }
    training_set = QADataset(train_json_path, tokenizer, max_source_length, max_target_length)
    training_loader = DataLoader(training_set, **train_params)
    print("Start Load Validation Data...")
    val_params = {
        "batch_size": batch_size,
        "shuffle": False,
        "num_workers": 0,
    }
    val_set = QADataset(val_json_path, tokenizer, max_source_length, max_target_length)
    val_loader = DataLoader(val_set, **val_params)


    scaler = torch.cuda.amp.GradScaler()
    optimizer = torch.optim.AdamW(params=model.parameters(), lr=lr)
    model = model.to(device)
    print("Start Training...")
    for epoch in range(epochs):
        train(epoch, model, device, training_loader, optimizer,scaler, gradient_accumulation_steps,model_output_dir)
        print("Save Model To ", model_output_dir)
        model.save_pretrained(model_output_dir)
    # 验证
    print("Start Validation...")
    with torch.no_grad():
        predictions, actuals = validate(tokenizer, model, device, val_loader, max_target_length)
        # 验证结果存储
        final_df = pd.DataFrame({"Generated Text": predictions, "Actual Text": actuals})
        val_data_path = os.path.join(model_output_dir, "predictions.csv")
        final_df.to_csv(val_data_path)
        print("Validation Data To ", val_data_path)


if __name__ == '__main__':
    main()

 

torch官方的自动混合精度代码

# Creates model and optimizer in default precision
model = Net().cuda()
optimizer = optim.SGD(model.parameters(), ...)

# Creates a GradScaler once at the beginning of training.
scaler = GradScaler()

for epoch in epochs:
    for input, target in data:
        optimizer.zero_grad()

        # Runs the forward pass with autocasting.
        with autocast(device_type='cuda', dtype=torch.float16):
            output = model(input)
            loss = loss_fn(output, target)

        # Scales loss.  Calls backward() on scaled loss to create scaled gradients.
        # Backward passes under autocast are not recommended.
        # Backward ops run in the same dtype autocast chose for corresponding forward ops.
        scaler.scale(loss).backward()

        # scaler.step() first unscales the gradients of the optimizer's assigned params.
        # If these gradients do not contain infs or NaNs, optimizer.step() is then called,
        # otherwise, optimizer.step() is skipped.
        scaler.step(optimizer)

        # Updates the scale for next iteration.
        scaler.update()

 

qlora在batch=16  lora_rank = 32 target_modules=["query_key_value","dense","dense_h_to_4h","dense_4h_to_h"]  对所有模型矩阵参数都进行lora的情况下才占用不足29g显存

而lora在batch=1 lora_rank = 8  只对query_key_value矩阵进行lora训练的情况下就占用了30g显存，而且速度会比较慢

 

 

报错 ValueError: Attempting to unscale FP16 gradients.

官方说不需要手动再调用.half()分别在模型和数据上

# model = model.half()

 

使用官方的混合精度训练，模型的loss确实可以稳定下降

 

7.使用qlora + torch fp16混合精度训练可以稳定，模型输出结果：

Qlora 后chatGLM6b 预测结果：

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14

[Round 1]   问：在上海的苹果代工厂，较低的基本工资让工人们形成了“软强制”的加班默契。加班能多拿两三千，“自愿”加班成为常态。律师提示，加班后虽能获得一时不错的报酬，但过重的工作负荷会透支身体，可能对今后劳动权利造成不利影响。 输出摘要：   答： --**Summary**-- 苹果代工厂员工调查：为何争着“自愿”加班 [Round 1]   问：上联：把酒邀春，春日三人醉 下联：   答： --**Couplet**-- 梳妆佩玉，玉王点一娇

　　actual label

?

1 2 3 4

--**Summary**-- 苹果代工厂员工调查：为何争着“自愿”加班 --**Couplet**-- 梳妆佩玉，玉王点一娇

　　

8.使用qlora加载模型推理 model_test.py  qlora推理占用的显存同样很低

 

 

from transformers import AutoTokenizer, AutoModel, AutoConfig,BitsAndBytesConfig
from peft import PeftConfig, PeftModel, LoraConfig, get_peft_model, TaskType
import torch
from transformers import (
    set_seed,
    HfArgumentParser,
    TrainingArguments,
    AutoModelForCausalLM
)

device = torch.device("cuda:0")

model_name = "/data/tmp/chatGLM2_6b_pretrain"
lora_dir = "output"

# 加载模型
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    device_map=0,
    load_in_4bit=True,
    torch_dtype=torch.float16,
    trust_remote_code=True,
    quantization_config=BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_compute_dtype=torch.float16,
        bnb_4bit_use_double_quant=True,
        bnb_4bit_quant_type="nf4",
        llm_int8_threshold=6.0,
        llm_int8_has_fp16_weight=False,
    ),
)

# peft_config = LoraConfig(
#     r=lora_rank,
#     lora_alpha=lora_alpha,
#     # target_modules=target_modules,
#     target_modules=["query_key_value","dense_h_to_4h"],
#     lora_dropout=lora_dropout,
#     bias="none",
#     task_type="CAUSAL_LM",
# )

# model = get_peft_model(model, peft_config)

# model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

config = PeftConfig.from_pretrained(lora_dir)
model = PeftModel.from_pretrained(model, lora_dir)

model = model.to(device)
model.eval()

while True:
    text = input("问题：")
    response, history = model.chat(tokenizer, text, history=[])
    print("回答：", response)
    

 

?

1 2 3

邓紫棋在北京鸟巢开演唱会，唱了音乐《 画》 。 请找出这段话中的实体 回答： --NER-- 北京鸟巢,邓紫棋