量化效果评估
======

用户如果要测试量化后模型效果，可参考插件方法，对浮点模型插入量化算子进行推理即可。 模型推理框架和输入不变。

**插件方法：**

以下为推理时对浮点模型插入量化算子，转换浮点数到量化模型的示例：

import sys
import torch

sys.path.append("path/to/dopt\_torch\_py3")

def get\_quanted\_model(base\_model, dopt\_config, quanted\_ckpt):
    from dopt.dopt\_lm.opt\_main import (optimize\_model, set\_quant\_state, set\_calibrate\_state, set\_run\_mode,)
    model = optimize\_model(base\_model, dopt\_config)
    model.load\_state\_dict(torch.load(quanted\_ckpt, map\_location=torch.device('cpu')), strict=True)
    set\_quant\_state(model, weight\_state=True, input\_state=True)
    set\_calibrate\_state(model, False)
    model.eval()
    return model

其中，参数的定义分别为：

*   base\_model：浮点模型定义，加载自AutoModelForCausalLM等接口。
    
*   dopt\_config：量化配置文件。
    
*   quanted\_ckpt：量化后的pth文件。
    

**插件方法推理完整示例**：

以下提供qwen2量化推理完整示例供用户参考，在该示例中添加了插件方法去验证量化效果，以量化工具包和量化文件作为输入，进行仿真推理，如果推理结果正常，说明量化成功。

import os
import sys
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

# 填写DDK\_tools工具包中tools\_dopt/dopt\_pytorch\_py3的真实路径
sys.path.append('path/to.dopt')
os.environ\["CUDA\_VISIBLE\_DEVICES"\] = "0"

def get\_quanted\_model(base\_model, dopt\_config, quanted\_ckpt):
    from dopt.dopt\_lm.opt\_main import (optimize\_model, set\_quant\_state, set\_calibrate\_state, set\_run\_mode,)
    model = optimize\_model(base\_model, dopt\_config)
    model.load\_state\_dict(torch.load(quanted\_ckpt, map\_location=torch.device('cpu')), strict=True)
    set\_quant\_state(model, weight\_state=True, input\_state=True)
    set\_calibrate\_state(model, False)
    model.eval()
    return model

def generate(prompt="Give me a short introduction to large language model."):
    messages = \[
        {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant"},
        {"role": "user", "content": prompt}
    \]
    text = tokenizer.apply\_chat\_template(messages, tokenize=False, add\_generation\_prompt=True)
    model\_inputs = tokenizer(\[text\], return\_tensors="pt").to(model.device)
    print(model\_inputs)
    generated\_ids = model.generate(\*\*model\_inputs, max\_new\_tokens=512)
    generated\_ids = \[
        output\_ids\[len(input\_ids):\] for input\_ids, output\_ids in zip(model\_inputs.input\_ids, generated\_ids)
    \]
    response = tokenizer.batch\_decode(generated\_ids, skip\_special\_tokens=True)\[0\]
    return response

if \_\_name\_\_ == '\_\_main\_\_':
    # 填写原模型的真实路径
    model\_name = "path/to/model"
    model = AutoModelForCausalLM.from\_pretrained(
        model\_name,
        torch\_dtype=torch.float16,
        device\_map="auto"
    )
    tokenizer = AutoTokenizer.from\_pretrained(model\_name)
    quant\_res\_root = 'you\_output\_path'
    # 量化配置
    dopt\_config = f"/{quant\_res\_root}/dopt\_config.json"
    # 量化权重
    quanted\_ckpt = f"/{quant\_res\_root}/train\_output/trained.pth"
    model = get\_quanted\_model(
        model,
        dopt\_config,
        quanted\_ckpt
    )
    prompt = "who are you?"
    response = generate(prompt)
    print(response)