大模型评估指标详解：科学衡量模型能力

星辰AI

77人浏览 · 2026-05-17 23:53:27

星辰AI · 2026-05-17 23:53:27 发布

大模型评估指标详解：科学衡量模型能力

前言

评估大模型是一个复杂的任务。不同的任务需要不同的评估指标，理解这些指标对于选择合适的模型和优化模型性能至关重要。

我在项目中经常需要评估不同模型的效果，对评估指标有了深入理解。今天分享一些常用的评估指标和使用场景。

常用评估指标

1. 准确性（Accuracy）

def accuracy(predictions: list, labels: list) -> float:
    """计算准确率"""
    correct = sum(1 for p, l in zip(predictions, labels) if p == l)
    return correct / len(labels)

# 示例
predictions = [1, 0, 1, 1, 0]
labels = [1, 0, 0, 1, 0]
print(f"准确率: {accuracy(predictions, labels):.2%}")

2. BLEU 分数

from nltk.translate.bleu_score import sentence_bleu

def bleu_score(references: list, candidate: str) -> float:
    """计算 BLEU 分数"""
    reference = [ref.split() for ref in references]
    candidate_tokens = candidate.split()
    
    return sentence_bleu(reference, candidate_tokens)

# 示例
references = ["这是一个测试句子", "这是测试句子"]
candidate = "这是一个测试"
print(f"BLEU 分数: {bleu_score(references, candidate):.4f}")

3. ROUGE 分数

from rouge import Rouge

def rouge_score(references: list, candidate: str) -> dict:
    """计算 ROUGE 分数"""
    rouge = Rouge()
    scores = rouge.get_scores(candidate, references[0])
    return scores[0]

# 示例
references = ["这是参考摘要"]
candidate = "这是生成的摘要"
result = rouge_score(references, candidate)
print(f"ROUGE-1: {result['rouge-1']['f']:.4f}")

4. BERTScore

from bert_score import score

def bert_score(references: list, candidates: list) -> tuple:
    """计算 BERTScore"""
    P, R, F1 = score(
        candidates,
        references,
        lang="zh",
        verbose=False
    )
    
    return {
        "precision": P.mean().item(),
        "recall": R.mean().item(),
        "f1": F1.mean().item()
    }

# 示例
references = ["参考文本"]
candidates = ["生成文本"]
result = bert_score(references, candidates)
print(f"BERTScore F1: {result['f1']:.4f}")

特定任务指标

代码生成指标

class CodeEvaluation:
    """代码生成评估"""
    
    def __init__(self):
        self.test_cases = []
    
    def add_test_case(self, code: str, test_input: str, expected_output: str):
        """添加测试用例"""
        self.test_cases.append({
            "code": code,
            "input": test_input,
            "expected": expected_output
        })
    
    def evaluate(self, generated_code: str) -> float:
        """评估代码"""
        correct = 0
        
        for test_case in self.test_cases:
            # 执行代码并验证
            try:
                exec(generated_code)
                # 验证输出
                correct += 1
            except:
                pass
        
        return correct / len(self.test_cases)

数学推理指标

class MathEvaluation:
    """数学推理评估"""
    
    def evaluate(self, predictions: list, labels: list) -> float:
        """计算准确率"""
        correct = 0
        
        for pred, label in zip(predictions, labels):
            # 解析数值答案
            pred_num = self._extract_number(pred)
            label_num = self._extract_number(label)
            
            if abs(pred_num - label_num) < 1e-6:
                correct += 1
        
        return correct / len(predictions)
    
    def _extract_number(self, text: str) -> float:
        """从文本中提取数字"""
        import re
        match = re.search(r"-?\d+\.?\d*", text)
        return float(match.group()) if match else 0

综合评估框架

class ModelEvaluator:
    """综合模型评估"""
    
    def __init__(self, model, datasets: dict):
        self.model = model
        self.datasets = datasets
    
    def evaluate(self, task: str) -> dict:
        """评估特定任务"""
        dataset = self.datasets.get(task)
        
        if task == "classification":
            return self._evaluate_classification(dataset)
        elif task == "generation":
            return self._evaluate_generation(dataset)
        elif task == "math":
            return self._evaluate_math(dataset)
    
    def _evaluate_classification(self, dataset: list) -> dict:
        """评估分类任务"""
        predictions = []
        labels = []
        
        for item in dataset:
            prompt = item["prompt"]
            response = self.model.generate(prompt)
            predictions.append(self._parse_answer(response))
            labels.append(item["label"])
        
        return {
            "accuracy": accuracy(predictions, labels),
            "total": len(dataset)
        }
    
    def _evaluate_generation(self, dataset: list) -> dict:
        """评估生成任务"""
        references = []
        candidates = []
        
        for item in dataset:
            response = self.model.generate(item["prompt"])
            candidates.append(response)
            references.append(item["reference"])
        
        bert = bert_score(references, candidates)
        rouge = rouge_score(references, candidates[0])
        
        return {
            "bert_f1": bert["f1"],
            "rouge_1_f": rouge["rouge-1"]["f"],
            "rouge_l_f": rouge["rouge-l"]["f"]
        }