模型量化完全指南【2026】- 从原理到部署实践

┌─────────────────────────────────────────────────────┐
│              量化精度与效率权衡                       │
├─────────────────────────────────────────────────────┤
│  精度类型  │ 比特位 │ 模型大小 │ 推理速度 │ 精度损失  │
├─────────────────────────────────────────────────────┤
│  FP32     │  32   │  100%   │  基准   │   无     │
│  FP16     │  16   │   50%   │  ~2x   │  极小    │
│  BF16     │  16   │   50%   │  ~2x   │  极小    │
│  INT8     │   8   │   25%   │  ~3x   │   小     │
│  INT4     │   4   │  12.5%  │  ~4x   │   中     │
│  INT2     │   2   │  6.25%  │  ~5x   │   大     │
└─────────────────────────────────────────────────────┘

import torch

fp32_tensor = torch.randn(1000, 1000, dtype=torch.float32)
fp16_tensor = fp32_tensor.to(torch.float16)
bf16_tensor = fp32_tensor.to(torch.bfloat16)

print(f"FP32 内存: {fp32_tensor.element_size() * fp32_tensor.numel() / 1024:.2f} KB")
print(f"FP16 内存: {fp16_tensor.element_size() * fp16_tensor.numel() / 1024:.2f} KB")
print(f"BF16 内存: {bf16_tensor.element_size() * bf16_tensor.numel() / 1024:.2f} KB")

q = round(x / scale)
x_dequant = q * scale

q = round(x / scale) + zero_point
x_dequant = (q - zero_point) * scale

from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig

model_id = "meta-llama/Llama-2-7b-hf"
tokenizer = AutoTokenizer.from_pretrained(model_id)

gptq_config = GPTQConfig(
    bits=4,
    dataset="c4",
    tokenizer=tokenizer,
    group_size=128,
    desc_act=True,
)

model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=gptq_config,
    device_map="auto",
)

model.save_pretrained("./llama-2-7b-gptq")

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = "meta-llama/Llama-2-7b-hf"
quant_path = "./llama-2-7b-awq"

model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)

quant_config = {
    "zero_point": True,
    "q_group_size": 128,
    "w_bit": 4,
    "version": "GEMM"
}

model.quantize(tokenizer, quant_config=quant_config)
model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)

git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
make -j

make LLAMA_CUBLAS=1 -j

python convert_hf_to_gguf.py /path/to/llama-2-7b --outfile llama-2-7b-f16.gguf --outtype f16

./llama-quantize llama-2-7b-f16.gguf llama-2-7b-q4_k_m.gguf q4_k_m

./llama-cli -m llama-2-7b-q4_k_m.gguf \
    -p "What is machine learning?" \
    -n 256 \
    --temp 0.7 \
    --top-p 0.9

from llama_cpp import Llama

llm = Llama(
    model_path="./llama-2-7b-q4_k_m.gguf",
    n_ctx=2048,
    n_threads=8,
    n_gpu_layers=35,
)

output = llm(
    "Q: What is model quantization?\nA:",
    max_tokens=256,
    temperature=0.7,
    top_p=0.9,
    stop=["Q:", "\n\n"],
)

print(output["choices"][0]["text"])

pip install bitsandbytes transformers accelerate

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = "meta-llama/Llama-2-7b-hf"

bnb_config_8bit = BitsAndBytesConfig(
    load_in_8bit=True,
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config_8bit,
    device_map="auto",
)

tokenizer = AutoTokenizer.from_pretrained(model_id)

print(f"模型显存占用: {model_8bit.get_memory_footprint() / 1024**3:.2f} GB")

bnb_config_4bit = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_use_double_quant=True,
)

model_4bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config_4bit,
    device_map="auto",
)

print(f"模型显存占用: {model_4bit.get_memory_footprint() / 1024**3:.2f} GB")

def generate_text(model, tokenizer, prompt, max_length=256):
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_length,
            temperature=0.7,
            top_p=0.9,
            do_sample=True,
            pad_token_id=tokenizer.eos_token_id,
        )
    
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    return response

prompt = "解释什么是模型量化："
result = generate_text(model_4bit, tokenizer, prompt)
print(result)

from datasets import load_dataset
from evaluate import load
import numpy as np

def evaluate_quantized_model(model, tokenizer, dataset, num_samples=100):
    perplexity_metric = load("perplexity", module_type="metric")
    
    texts = dataset["text"][:num_samples]
    
    results = perplexity_metric.compute(
        predictions=texts,
        model_id=model,
        add_start_token=False,
    )
    
    return results["mean_perplexity"]

def compare_models(original_model, quantized_model, tokenizer, test_prompts):
    results = []
    
    for prompt in test_prompts:
        original_output = generate_text(original_model, tokenizer, prompt)
        quantized_output = generate_text(quantized_model, tokenizer, prompt)
        
        results.append({
            "prompt": prompt,
            "original": original_output,
            "quantized": quantized_output,
        })
    
    return results

from vllm import LLM, SamplingParams

llm = LLM(
    model="TheBloke/Llama-2-7B-GPTQ",
    quantization="gptq",
    dtype="float16",
    gpu_memory_utilization=0.9,
)

sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=256,
)

prompts = ["What is artificial intelligence?"]
outputs = llm.generate(prompts, sampling_params)

for output in outputs:
    print(output.outputs[0].text)

ollama create mymodel -f Modelfile

ollama run mymodel "What is model quantization?"

FROM ./llama-2-7b-q4_k_m.gguf

PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER num_ctx 2048

SYSTEM You are a helpful AI assistant.