LoRA微调技术完全指南【2026】- 低秩适应原理与实践

W' = W + ΔW

ΔW = B × A

from peft import PeftModel

base_model = load_base_model()

model_task_a = PeftModel.from_pretrained(base_model, "lora-adapter-task-a")

model_task_b = PeftModel.from_pretrained(base_model, "lora-adapter-task-b")

┌─────────────────────────────────────────────────┐
│              rank 参数选择指南                    │
├─────────────────────────────────────────────────┤
│  rank值  │  参数量   │  适用场景                  │
├─────────────────────────────────────────────────┤
│    4     │   最少    │  简单任务、快速实验         │
│    8     │    少     │  一般任务、资源受限         │
│   16     │   中等    │  推荐默认值、平衡选择       │
│   32     │    多     │  复杂任务、追求效果         │
│   64     │   较多    │  高复杂度、接近全量微调     │
│  128+    │   很多    │  特殊需求、充足资源         │
└─────────────────────────────────────────────────┘

ΔW = (alpha / rank) × B × A

lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
)

target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]

target_modules = ["c_attn", "c_proj", "c_fc"]

lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    lora_dropout=0.05,
)

pip install torch transformers datasets peft accelerate bitsandbytes
pip install trl

import torch
from datasets import load_dataset
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    TrainingArguments,
    BitsAndBytesConfig,
)
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer

model_name = "Qwen/Qwen2-7B"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True,
)

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"

model = prepare_model_for_kbit_training(model)

lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj",
    ],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()

dataset = load_dataset("json", data_files="train_data.json", split="train")

def formatting_func(example):
    text = f"""<|im_start|>system
你是一个专业的AI助手。<|im_end|>
<|im_start|>user
{example['instruction']}<|im_end|>
<|im_start|>assistant
{example['output']}<|im_end|>"""
    return text

training_args = TrainingArguments(
    output_dir="./qwen-lora",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    warmup_ratio=0.03,
    logging_steps=10,
    save_strategy="epoch",
    bf16=True,
    optim="paged_adamw_8bit",
    gradient_checkpointing=True,
    max_grad_norm=0.3,
)

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    formatting_func=formatting_func,
    max_seq_length=1024,
    args=training_args,
)

trainer.train()

model.save_pretrained("./qwen-lora-adapter")
tokenizer.save_pretrained("./qwen-lora-adapter")

model.print_trainable_parameters()

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

base_model = AutoModelForCausalLM.from_pretrained(
    "Qwen/Qwen2-7B",
    torch_dtype=torch.float16,
    device_map="auto",
    trust_remote_code=True,
)

model = PeftModel.from_pretrained(base_model, "./qwen-lora-adapter")

merged_model = model.merge_and_unload()

merged_model.save_pretrained("./qwen-merged")
tokenizer = AutoTokenizer.from_pretrained("./qwen-lora-adapter")
tokenizer.save_pretrained("./qwen-merged")

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

model = AutoModelForCausalLM.from_pretrained(
    "./qwen-merged",
    torch_dtype=torch.float16,
    device_map="auto",
)
tokenizer = AutoTokenizer.from_pretrained("./qwen-merged")

def generate(prompt, max_new_tokens=256):
    messages = [
        {"role": "system", "content": "你是一个专业的AI助手。"},
        {"role": "user", "content": prompt}
    ]
    
    text = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
    
    inputs = tokenizer(text, return_tensors="pt").to(model.device)
    
    outputs = model.generate(
        **inputs,
        max_new_tokens=max_new_tokens,
        temperature=0.7,
        top_p=0.9,
        do_sample=True,
    )
    
    response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
    return response

result = generate("解释一下什么是机器学习？")
print(result)

from peft import PeftModel

base_model = load_base_model()

model = PeftModel.from_pretrained(base_model, "./adapter-task-a")
response_a = generate(model, prompt)

model.load_adapter("./adapter-task-b", adapter_name="task_b")
model.set_adapter("task_b")
response_b = generate(model, prompt)