小模型崛起：2B/8B 参数模型如何在边缘设备上替代大模型

# 使用 Ollama 运行 Phi-4 Mini
import requests

response = requests.post("http://localhost:11434/api/generate", json={
    "model": "phi4-mini",
    "prompt": "用Python实现一个高效的LRU缓存，要求O(1)时间复杂度",
    "stream": False
})
print(response.json()["response"])

# 安装 Ollama
curl -fsSL https://ollama.com/install.sh | sh

# 下载并运行 Phi-4 Mini (量化版约 2.5GB)
ollama pull phi4-mini
ollama run phi4-mini

# 下载 Gemma 3 4B
ollama pull gemma3:4b

# 下载 Qwen3 4B
ollama pull qwen3:4b

# 查看已下载的模型
ollama list

import { CreateMLCEngine } from "@mlc-ai/web-llm";

// 在浏览器中加载 Gemma 3 1B 模型
const engine = await CreateMLCEngine("gemma-3-1b-it-q4f16_1-MLC", {
  initProgressCallback: (progress) => {
    console.log(`模型加载进度: ${(progress.progress * 100).toFixed(1)}%`);
  }
});

// 进行推理
const reply = await engine.chat.completions.create({
  messages: [{ role: "user", content: "解释什么是边缘计算" }],
  temperature: 0.7,
  max_tokens: 512
});
console.log(reply.choices[0].message.content);

# 使用 llama.cpp 在树莓派 5 上运行 Qwen3.5-2B
./llama-server \
  -m qwen3.5-2b-q4_k_m.gguf \
  --host 0.0.0.0 \
  --port 8080 \
  -ngl 0 \
  -c 2048 \
  -t 4

# 使用 llama.cpp 将 HuggingFace 模型转换为 GGUF 格式
python convert_hf_to_gguf.py \
  ./Qwen3-4B \
  --outfile qwen3-4b-f16.gguf \
  --outtype f16

# 执行 INT4 量化
./llama-quantize \
  qwen3-4b-f16.gguf \
  qwen3-4b-q4_k_m.gguf \
  Q4_K_M

# 量化前后体积对比
# FP16:  ~8.0 GB
# Q4_K_M: ~2.5 GB  (压缩率 68%)

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer, SFTConfig

# 1. 加载模型（4-bit 量化）
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype="bfloat16",
    bnb_4bit_use_double_quant=True
)

model = AutoModelForCausalLM.from_pretrained(
    "Qwen/Qwen3-4B",
    quantization_config=bnb_config,
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-4B")

# 2. 配置 LoRA
lora_config = LoraConfig(
    r=16,                          # 小模型用 r=16 即可
    lora_alpha=32,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM"
)

model = prepare_model_for_kbit_training(model)
model = get_peft_model(model, lora_config)

# 可训练参数仅占总参数的 0.4%
model.print_trainable_parameters()
# 输出: trainable params: 16,384,000 || all params: 4,000,000,000 || 0.41%

# 3. 训练配置
training_config = SFTConfig(
    output_dir="./qwen3-4b-lora",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    bf16=True,
    logging_steps=10,
    save_strategy="epoch"
)

# 4. 开始训练（约 30 分钟 / RTX 4060 8GB）
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    args=training_config,
    tokenizer=tokenizer
)
trainer.train()

import requests
import json
from typing import Generator

class LocalLLMService:
    """基于 Ollama 的本地 LLM 推理服务"""

    def __init__(self, base_url: str = "http://localhost:11434"):
        self.base_url = base_url

    def generate(self, prompt: str, model: str = "phi4-mini",
                 temperature: float = 0.7) -> str:
        """同步生成"""
        response = requests.post(
            f"{self.base_url}/api/generate",
            json={
                "model": model,
                "prompt": prompt,
                "temperature": temperature,
                "stream": False
            }
        )
        return response.json()["response"]

    def stream_generate(self, prompt: str, model: str = "phi4-mini",
                        temperature: float = 0.7) -> Generator[str, None, None]:
        """流式生成"""
        response = requests.post(
            f"{self.base_url}/api/generate",
            json={
                "model": model,
                "prompt": prompt,
                "temperature": temperature,
                "stream": True
            },
            stream=True
        )
        for line in response.iter_lines():
            if line:
                data = json.loads(line)
                if not data.get("done"):
                    yield data["response"]

    def chat(self, messages: list, model: str = "phi4-mini") -> str:
        """多轮对话"""
        response = requests.post(
            f"{self.base_url}/api/chat",
            json={
                "model": model,
                "messages": messages,
                "stream": False
            }
        )
        return response.json()["message"]["content"]


# 使用示例
service = LocalLLMService()

# 场景 1：代码审查
review = service.generate(
    "Review this Python function for bugs and improvements:\n"
    "def calc(x): return x*x if x>0 else -x",
    model="phi4-mini"
)
print("代码审查结果:", review)

# 场景 2：中文客服意图识别
intent = service.generate(
    "判断以下客户消息的意图类别（退款/咨询/投诉/表扬）:\n"
    "我上周买的东西到现在还没到，你们到底什么时候发货？",
    model="qwen3:4b"
)
print("意图识别:", intent)

# 场景 3：流式输出
print("流式生成: ", end="")
for token in service.stream_generate("用三句话解释量子计算"):
    print(token, end="", flush=True)

任务是否明确且可定义？
  ├── 是 → 微调小模型（2B-8B + LoRA）
  │        ├── 需要离线/隐私 → Ollama 本地部署
  │        ├── 需要浏览器端 → WebLLM
  │        └── 需要手机端 → llama.cpp / CoreML
  └── 否 → 大模型 API
           ├── 高并发 → GPT-4o-mini / Claude Haiku
           └── 高质量 → GPT-4o / Claude Opus