從菜鳥到出師
NLP 四層學習地圖

# Attention 機制的直覺：對每個詞，計算它和其他詞的「相關度」
# 數學上就是：Attention(Q,K,V) = softmax(QK^T / √d_k) × V

import torch
import torch.nn.functional as F

d_k = 64   # 向量維度

# 假設 4 個詞，每個詞是 64 維向量
Q = torch.randn(4, d_k)   # Query：「我想找什麼？」
K = torch.randn(4, d_k)   # Key：「我有什麼？」
V = torch.randn(4, d_k)   # Value：「實際的內容」

# 計算注意力分數（相關度矩陣）
scores  = Q @ K.T / d_k**0.5   # shape: (4, 4)
weights = F.softmax(scores, dim=-1)  # 每行加總為 1
output  = weights @ V              # 加權平均所有詞的 Value

# → 「我」這個詞，透過 attention 知道它和「書」最相關
# → 所以「我」的新向量大量融合了「書」的資訊
# → 這就是「上下文感知」的來源！

print(f"注意力權重矩陣（4×4）:\n{weights.detach().numpy().round(3)}")

from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    TrainingArguments, Trainer, DataCollatorWithPadding
)
from datasets import Dataset
import numpy as np
from sklearn.metrics import classification_report

# ── Step 1：準備資料 ──
# 假設 df 有 'text' 和 'label'（整數）兩欄
train_df, val_df, test_df = ...  # 切分資料

# ── Step 2：Tokenize ──
MODEL_NAME = "bert-base-chinese"
tokenizer  = AutoTokenizer.from_pretrained(MODEL_NAME)

def tokenize(batch):
    return tokenizer(batch["text"], truncation=True, max_length=128)

train_ds = Dataset.from_pandas(train_df).map(tokenize, batched=True)
val_ds   = Dataset.from_pandas(val_df  ).map(tokenize, batched=True)
test_ds  = Dataset.from_pandas(test_df ).map(tokenize, batched=True)

# ── Step 3：載入模型（自動加分類頭）──
num_labels = train_df["label"].nunique()
model = AutoModelForSequenceClassification.from_pretrained(
    MODEL_NAME, num_labels=num_labels
)

# ── Step 4：訓練設定 ──
args = TrainingArguments(
    output_dir         = "./results",
    num_train_epochs   = 3,
    per_device_train_batch_size = 16,
    per_device_eval_batch_size  = 32,
    evaluation_strategy = "epoch",
    save_strategy       = "epoch",
    load_best_model_at_end = True,
    metric_for_best_model  = "f1",   # ← 用 F1，不用 Accuracy！
    learning_rate   = 2e-5,        # ← Fine-tuning 標準學習率
    warmup_ratio    = 0.1,
    weight_decay    = 0.01,
    fp16            = True,         # GPU 加速（Colab 適用）
)

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    report = classification_report(
        labels, preds, output_dict=True, zero_division=0)
    return {"f1": report["weighted avg"]["f1-score"]}

trainer = Trainer(
    model           = model,
    args            = args,
    train_dataset   = train_ds,
    eval_dataset    = val_ds,
    data_collator   = DataCollatorWithPadding(tokenizer),
    compute_metrics = compute_metrics,
)

# 開始訓練！（T4 GPU 約 3 分鐘/epoch × 3 = 9 分鐘）
trainer.train()

# 測試集評估
preds_output = trainer.predict(test_ds)
preds = np.argmax(preds_output.predictions, axis=-1)
print(classification_report(test_ds["label"], preds, zero_division=0))

from sklearn.metrics import (
    classification_report, confusion_matrix,
    ConfusionMatrixDisplay
)
from statsmodels.stats.contingency_tables import mcnemar
import matplotlib.pyplot as plt
import numpy as np

# ── 完整的 Classification Report ──
print(classification_report(
    y_true, y_pred,
    target_names=["客訴", "詢價", "閒聊", "商品評價"],
    digits=4   # ← 論文報告通常要 4 位小數
))
# precision  recall  f1-score   support
# 客訴         0.8912  0.9104  0.9007    67
# 詢價         0.7634  0.8021  0.7823    53
# 閒聊         0.9321  0.8876  0.9093    45
# 商品評價      0.9102  0.9231  0.9166   135

# ── 混淆矩陣視覺化 ──
cm = confusion_matrix(y_true, y_pred)
disp = ConfusionMatrixDisplay(cm, display_labels=["客訴","詢價","閒聊","商品評價"])
disp.plot(cmap="Blues")
plt.title("混淆矩陣 — 觀察哪些類別最常混淆")

# ── McNemar's Test：比較兩個模型 ──
# 零樣本 vs Fine-tuned 模型，哪個真的更好？
correct_a = (y_true == y_pred_zeroshot)   # 零樣本答對的
correct_b = (y_true == y_pred_finetuned)  # Fine-tuned 答對的

# 建立 2×2 contingency table
n_both_right   = (correct_a & correct_b).sum()
n_only_a_right = (correct_a & ~correct_b).sum()
n_only_b_right = (~correct_a & correct_b).sum()
n_both_wrong   = (~correct_a & ~correct_b).sum()

table = [[n_both_right, n_only_a_right],
         [n_only_b_right, n_both_wrong]]

result = mcnemar(table, exact=True)
print(f"McNemar p-value = {result.pvalue:.4f}")
# p < 0.05 → 兩模型有顯著差異，可以宣稱 Fine-tuned 更好
# p ≥ 0.05 → 差距可能只是隨機波動，不能宣稱誰更好

# ── Bootstrap Confidence Interval（測試集小時使用）──
def bootstrap_f1(y_true, y_pred, n_bootstrap=1000, ci=0.95):
    from sklearn.utils import resample
    from sklearn.metrics import f1_score
    scores = []
    for _ in range(n_bootstrap):
        yt, yp = resample(y_true, y_pred)
        scores.append(f1_score(yt, yp, average="weighted"))
    alpha = (1 - ci) / 2
    return np.quantile(scores, [alpha, 1-alpha])

lo, hi = bootstrap_f1(y_true, y_pred_finetuned)
print(f"Weighted F1: {f1:.4f} (95% CI: [{lo:.4f}, {hi:.4f}])")

# ═══════════════════════════════════════
# 概念 A：Few-shot Prompting
# ═══════════════════════════════════════
few_shot_prompt = """
請判斷以下評論的類別（客訴/詢價/閒聊/商品評價）。

範例：
評論：這個商品品質很差，要求退款！ → 客訴
評論：請問有黑色款嗎？            → 詢價
評論：今天天氣很好。              → 閒聊
評論：超好用，強力推薦！          → 商品評價

請判斷：
評論：{text}                    →"""

# 對比零樣本：few-shot 給了模型「格式範例」，通常準確率更高


# ═══════════════════════════════════════
# 概念 B：RAG — 你的 Embedding 技術直接用得上
# ═══════════════════════════════════════
from sklearn.metrics.pairwise import cosine_similarity

# 假設你有一批領域文件的 BERT Embedding（就是 Module 2 的成果）
doc_embeddings = state.embeddings["bert"]  # shape: (n_docs, 768)
doc_texts      = df["review"].tolist()

def rag_retrieve(query, top_k=3):
    # 問題也轉成 Embedding
    query_emb = get_bert_embeddings([query], "bert-base-chinese")
    # 找最相似的文件
    sims = cosine_similarity(query_emb, doc_embeddings)[0]
    top_idx = sims.argsort()[::-1][:top_k]
    return [(doc_texts[i], sims[i]) for i in top_idx]

# 這就是 RAG 的核心：用相似度檢索 → 餵給 LLM → 讓 LLM 基於真實資料回答
results = rag_retrieve("客服態度的問題")


# ═══════════════════════════════════════
# 概念 C：LoRA 微調（需安裝 peft）
# ═══════════════════════════════════════
# !pip install peft
from peft import get_peft_model, LoraConfig, TaskType

lora_config = LoraConfig(
    task_type = TaskType.SEQ_CLS,
    r         = 8,       # 低秩維度（越小越省，通常 4-16）
    lora_alpha = 32,     # 縮放係數
    target_modules = ["query", "value"],  # 只更新 Q、V 矩陣
    lora_dropout = 0.1,
)

# 把 LoRA 套在現有模型上：只有 ~0.5% 的參數需要訓練！
peft_model = get_peft_model(model, lora_config)
peft_model.print_trainable_parameters()
# trainable params: 294,912 || all params: 102,267,648 (0.29%)
# → 只需訓練 0.29% 的參數，效果接近 Full Fine-tuning！