AI内容检测技术：基于机器学习的文本识别与标记系统实现

weixin_34289454

355人浏览 · 2026-07-14 15:43:06

weixin_34289454 · 2026-07-14 15:43:06 发布

1. 背景与核心概念

最近在技术社区中，关于AI生成内容的识别与标记成为了热门话题。Hacker News社区有用户提议为AI生成文章添加标记功能，这反映了当前技术领域对内容真实性和来源透明性的重视。作为开发者，我们不仅需要关注这一趋势，更应该掌握实际的技术实现方案。

AI内容识别技术主要解决以下几个核心问题：

内容真实性验证 ：帮助读者区分人工创作和机器生成的内容
版权保护 ：确保原创内容的权益得到保护
信息可信度评估 ：为内容质量判断提供参考依据
技术透明度 ：促进AI技术的健康发展

在实际开发中，我们可以通过多种技术手段来实现内容识别和标记功能。本文将重点介绍基于机器学习和自然语言处理的实用解决方案，包含完整的代码实现和部署方案。

2. 技术方案选型与环境准备

2.1 技术栈选择

根据实际需求和技术成熟度，我们选择以下技术栈：

编程语言 ：Python 3.8+
机器学习框架 ：PyTorch 1.12+
自然语言处理库 ：Transformers 4.20+
Web框架 ：FastAPI 0.85+
数据库 ：PostgreSQL 14+

2.2 环境配置要求

# 创建虚拟环境
python -m venv ai_detection_env
source ai_detection_env/bin/activate  # Linux/Mac
# ai_detection_env\Scripts\activate  # Windows

# 安装核心依赖
pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113
pip install transformers==4.20.1
pip install fastapi==0.85.0 uvicorn==0.18.3
pip install sqlalchemy==1.4.41 psycopg2-binary==2.9.3
pip install scikit-learn==1.1.2 pandas==1.4.3

2.3 项目结构设计

ai_content_detector/
├── app/
│   ├── __init__.py
│   ├── main.py           # FastAPI主应用
│   ├── models/           # 数据模型
│   │   ├── __init__.py
│   │   └── detection.py
│   ├── services/         # 业务逻辑
│   │   ├── __init__.py
│   │   └── detector.py
│   └── utils/           # 工具函数
│       ├── __init__.py
│       └── preprocess.py
├── config/
│   └── settings.py      # 配置文件
├── requirements.txt
└── README.md

3. 核心检测算法原理

3.1 文本特征提取

AI生成文本与人工创作文本在语言模式上存在显著差异。我们主要关注以下特征维度：

# 文件路径：app/utils/preprocess.py
import re
from collections import Counter
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer

class TextFeatureExtractor:
    def __init__(self):
        self.vectorizer = TfidfVectorizer(
            max_features=5000,
            ngram_range=(1, 3),
            stop_words='english'
        )
    
    def extract_linguistic_features(self, text):
        """提取语言学特征"""
        features = {}
        
        # 文本长度特征
        features['char_count'] = len(text)
        features['word_count'] = len(text.split())
        features['sentence_count'] = len(re.split(r'[.!?]+', text))
        
        # 词汇多样性
        words = text.lower().split()
        unique_words = set(words)
        features['lexical_diversity'] = len(unique_words) / len(words) if words else 0
        
        # 标点符号使用模式
        features['punctuation_ratio'] = len(re.findall(r'[^\w\s]', text)) / len(text) if text else 0
        
        return features
    
    def extract_semantic_features(self, text):
        """提取语义特征"""
        # 使用预训练的BERT模型获取语义嵌入
        from transformers import AutoTokenizer, AutoModel
        import torch
        
        tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
        model = AutoModel.from_pretrained('bert-base-uncased')
        
        inputs = tokenizer(text, return_tensors='pt', truncation=True, max_length=512)
        with torch.no_grad():
            outputs = model(**inputs)
        
        # 获取CLS token的嵌入作为文本表示
        embeddings = outputs.last_hidden_state[:, 0, :].numpy()
        return embeddings.flatten()

3.2 机器学习模型设计

基于提取的特征，我们构建一个集成分类器：

# 文件路径：app/services/detector.py
import numpy as np
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
import joblib

class AIContentDetector:
    def __init__(self):
        self.models = {}
        self.scaler = StandardScaler()
        self.is_trained = False
    
    def create_ensemble_model(self):
        """创建集成学习模型"""
        rf = RandomForestClassifier(
            n_estimators=100,
            max_depth=10,
            random_state=42
        )
        
        svm = SVC(
            kernel='rbf',
            C=1.0,
            probability=True,
            random_state=42
        )
        
        lr = LogisticRegression(
            C=0.1,
            max_iter=1000,
            random_state=42
        )
        
        ensemble = VotingClassifier(
            estimators=[
                ('rf', rf),
                ('svm', svm),
                ('lr', lr)
            ],
            voting='soft'
        )
        
        return ensemble
    
    def train(self, X, y):
        """训练模型"""
        # 数据标准化
        X_scaled = self.scaler.fit_transform(X)
        
        # 创建并训练集成模型
        self.model = self.create_ensemble_model()
        self.model.fit(X_scaled, y)
        self.is_trained = True
        
        return self
    
    def predict(self, X):
        """预测内容类型"""
        if not self.is_trained:
            raise ValueError("模型尚未训练")
        
        X_scaled = self.scaler.transform(X)
        predictions = self.model.predict_proba(X_scaled)
        
        return predictions[:, 1]  # 返回AI生成概率

4. 完整系统实现

4.1 数据库模型设计

# 文件路径：app/models/detection.py
from sqlalchemy import Column, Integer, String, DateTime, Float, Text
from sqlalchemy.ext.declarative import declarative_base
from datetime import datetime

Base = declarative_base()

class ContentAnalysis(Base):
    __tablename__ = 'content_analysis'
    
    id = Column(Integer, primary_key=True, index=True)
    content_hash = Column(String(64), unique=True, index=True)
    original_text = Column(Text, nullable=False)
    ai_probability = Column(Float, nullable=False)
    confidence_score = Column(Float, nullable=False)
    analysis_result = Column(String(20), nullable=False)  # 'AI_GENERATED' or 'HUMAN_WRITTEN'
    features_json = Column(Text)  # 存储提取的特征
    created_at = Column(DateTime, default=datetime.utcnow)
    
    def to_dict(self):
        return {
            'id': self.id,
            'content_hash': self.content_hash,
            'ai_probability': self.ai_probability,
            'confidence_score': self.confidence_score,
            'analysis_result': self.analysis_result,
            'created_at': self.created_at.isoformat()
        }

4.2 API接口实现

# 文件路径：app/main.py
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import Optional
import hashlib
from datetime import datetime

from app.services.detector import AIContentDetector
from app.utils.preprocess import TextFeatureExtractor
from app.models.detection import ContentAnalysis, Base
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

app = FastAPI(title="AI内容检测API", version="1.0.0")

# 数据库配置
SQLALCHEMY_DATABASE_URL = "postgresql://username:password@localhost/ai_detection"
engine = create_engine(SQLALCHEMY_DATABASE_URL)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

# 创建表
Base.metadata.create_all(bind=engine)

# 初始化组件
feature_extractor = TextFeatureExtractor()
detector = AIContentDetector()

class ContentRequest(BaseModel):
    text: str
    author: Optional[str] = None
    source: Optional[str] = None

class DetectionResponse(BaseModel):
    content_hash: str
    ai_probability: float
    confidence_score: float
    result: str
    analysis_time: datetime

@app.post("/detect", response_model=DetectionResponse)
async def detect_ai_content(request: ContentRequest):
    """检测内容是否为AI生成"""
    try:
        # 计算内容哈希
        content_hash = hashlib.sha256(request.text.encode()).hexdigest()
        
        # 检查是否已分析过
        db = SessionLocal()
        existing_analysis = db.query(ContentAnalysis).filter(
            ContentAnalysis.content_hash == content_hash
        ).first()
        
        if existing_analysis:
            return DetectionResponse(
                content_hash=existing_analysis.content_hash,
                ai_probability=existing_analysis.ai_probability,
                confidence_score=existing_analysis.confidence_score,
                result=existing_analysis.analysis_result,
                analysis_time=existing_analysis.created_at
            )
        
        # 提取特征
        linguistic_features = feature_extractor.extract_linguistic_features(request.text)
        semantic_features = feature_extractor.extract_semantic_features(request.text)
        
        # 合并特征（这里需要根据实际训练数据调整）
        features = np.concatenate([
            list(linguistic_features.values()),
            semantic_features
        ]).reshape(1, -1)
        
        # 预测
        ai_prob = detector.predict(features)[0]
        confidence = abs(ai_prob - 0.5) * 2  # 置信度计算
        result = "AI_GENERATED" if ai_prob > 0.5 else "HUMAN_WRITTEN"
        
        # 保存结果
        analysis = ContentAnalysis(
            content_hash=content_hash,
            original_text=request.text,
            ai_probability=ai_prob,
            confidence_score=confidence,
            analysis_result=result
        )
        
        db.add(analysis)
        db.commit()
        
        return DetectionResponse(
            content_hash=content_hash,
            ai_probability=ai_prob,
            confidence_score=confidence,
            result=result,
            analysis_time=datetime.utcnow()
        )
        
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"分析失败: {str(e)}")
    finally:
        db.close()

@app.get("/analysis/{content_hash}")
async def get_analysis(content_hash: str):
    """获取分析结果"""
    db = SessionLocal()
    analysis = db.query(ContentAnalysis).filter(
        ContentAnalysis.content_hash == content_hash
    ).first()
    
    if not analysis:
        raise HTTPException(status_code=404, detail="未找到分析记录")
    
    return analysis.to_dict()

4.3 前端标记组件

<!-- 文件路径：static/marker.js -->
class AIContentMarker {
    constructor(options = {}) {
        this.options = {
            threshold: 0.7,
            markerClass: 'ai-content-marker',
            ...options
        };
        this.init();
    }
    
    init() {
        this.createMarkerStyle();
        this.scanAndMark();
        
        // 监听动态内容变化
        const observer = new MutationObserver((mutations) => {
            mutations.forEach((mutation) => {
                if (mutation.addedNodes.length) {
                    this.scanAndMark();
                }
            });
        });
        
        observer.observe(document.body, {
            childList: true,
            subtree: true
        });
    }
    
    createMarkerStyle() {
        const style = document.createElement('style');
        style.textContent = `
            .${this.options.markerClass} {
                position: relative;
                border-left: 3px solid #ff6b6b;
                padding-left: 10px;
                margin: 10px 0;
            }
            
            .${this.options.markerClass}::before {
                content: "AI生成内容";
                position: absolute;
                top: -20px;
                left: 0;
                background: #ff6b6b;
                color: white;
                padding: 2px 8px;
                font-size: 12px;
                border-radius: 3px;
            }
            
            .ai-confidence-low {
                border-left-color: #ffa726;
            }
            
            .ai-confidence-low::before {
                content: "可能为AI生成";
                background: #ffa726;
            }
        `;
        document.head.appendChild(style);
    }
    
    async scanAndMark() {
        const contentElements = document.querySelectorAll('.content, .article, .post');
        
        for (const element of contentElements) {
            if (element.dataset.aiAnalyzed) continue;
            
            const text = element.textContent.trim();
            if (text.length < 100) continue; // 跳过过短内容
            
            try {
                const result = await this.analyzeContent(text);
                this.markElement(element, result);
                element.dataset.aiAnalyzed = 'true';
            } catch (error) {
                console.warn('内容分析失败:', error);
            }
        }
    }
    
    async analyzeContent(text) {
        const response = await fetch('/detect', {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
            },
            body: JSON.stringify({ text })
        });
        
        if (!response.ok) {
            throw new Error('分析请求失败');
        }
        
        return await response.json();
    }
    
    markElement(element, result) {
        if (result.ai_probability > this.options.threshold) {
            element.classList.add(this.options.markerClass);
            
            if (result.confidence_score < 0.8) {
                element.classList.add('ai-confidence-low');
            }
            
            // 添加详细提示
            const tooltip = this.createTooltip(result);
            element.appendChild(tooltip);
        }
    }
    
    createTooltip(result) {
        const tooltip = document.createElement('div');
        tooltip.className = 'ai-detection-tooltip';
        tooltip.innerHTML = `
            <div style="font-size: 12px; color: #666; margin-top: 5px;">
                AI生成概率: ${(result.ai_probability * 100).toFixed(1)}% |
                置信度: ${(result.confidence_score * 100).toFixed(1)}%
            </div>
        `;
        return tooltip;
    }
}

// 使用示例
document.addEventListener('DOMContentLoaded', () => {
    new AIContentMarker({
        threshold: 0.6,
        markerClass: 'ai-generated-content'
    });
});

5. 模型训练与优化

5.1 数据准备与预处理

# 文件路径：train_model.py
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from app.services.detector import AIContentDetector
from app.utils.preprocess import TextFeatureExtractor
import joblib

def prepare_training_data():
    """准备训练数据"""
    # 这里需要准备人工写作和AI生成的数据集
    # 实际项目中应从可靠来源获取
    
    human_texts = [
        "这是一篇人工撰写的技术文章...",
        # 更多人工写作样本
    ]
    
    ai_texts = [
        "根据当前技术发展趋势...",
        # 更多AI生成样本
    ]
    
    # 创建标签
    human_labels = [0] * len(human_texts)  # 人工写作标签为0
    ai_labels = [1] * len(ai_texts)        # AI生成标签为1
    
    texts = human_texts + ai_texts
    labels = human_labels + ai_labels
    
    return texts, labels

def extract_features(texts):
    """批量提取特征"""
    extractor = TextFeatureExtractor()
    features = []
    
    for text in texts:
        linguistic_feats = extractor.extract_linguistic_features(text)
        semantic_feats = extractor.extract_semantic_features(text)
        
        # 合并特征
        combined = np.concatenate([
            list(linguistic_feats.values()),
            semantic_feats
        ])
        features.append(combined)
    
    return np.array(features)

def train_detection_model():
    """训练检测模型"""
    print("准备训练数据...")
    texts, labels = prepare_training_data()
    
    print("提取特征...")
    features = extract_features(texts)
    
    print("划分训练测试集...")
    X_train, X_test, y_train, y_test = train_test_split(
        features, labels, test_size=0.2, random_state=42
    )
    
    print("训练模型...")
    detector = AIContentDetector()
    detector.train(X_train, y_train)
    
    # 评估模型
    from sklearn.metrics import classification_report, accuracy_score
    y_pred = detector.model.predict(detector.scaler.transform(X_test))
    
    print("模型准确率:", accuracy_score(y_test, y_pred))
    print("\n分类报告:")
    print(classification_report(y_test, y_pred))
    
    # 保存模型
    joblib.dump({
        'model': detector.model,
        'scaler': detector.scaler
    }, 'ai_detector_model.pkl')
    
    print("模型训练完成并保存")

if __name__ == "__main__":
    train_detection_model()

5.2 模型性能优化

# 文件路径：app/services/advanced_detector.py
import numpy as np
from transformers import pipeline
from sentence_transformers import SentenceTransformer
import torch.nn as nn
import torch

class AdvancedAIDetector:
    def __init__(self):
        self.sentence_model = SentenceTransformer('all-MiniLM-L6-v2')
        self.perplexity_analyzer = pipeline(
            "text-generation",
            model="gpt2",
            device=0 if torch.cuda.is_available() else -1
        )
    
    def calculate_perplexity(self, text):
        """计算文本困惑度"""
        try:
            # 使用GPT-2计算困惑度
            results = self.perplexity_analyzer(
                text,
                max_length=len(text.split()) + 10,
                return_full_text=False,
                num_return_sequences=1
            )
            
            # 简化计算，实际需要更复杂的困惑度计算
            return len(text.split()) / max(1, len(set(text.split())))
        except:
            return 0.0
    
    def analyze_writing_pattern(self, text):
        """分析写作模式"""
        sentences = text.split('.')
        features = {}
        
        if len(sentences) < 2:
            return features
        
        # 句子长度变化分析
        sentence_lengths = [len(s.split()) for s in sentences if s.strip()]
        features['length_variance'] = np.var(sentence_lengths)
        features['avg_sentence_length'] = np.mean(sentence_lengths)
        
        # 词汇重复模式
        words = text.lower().split()
        word_freq = {}
        for word in words:
            word_freq[word] = word_freq.get(word, 0) + 1
        
        # 计算重复率
        repeat_rate = sum(1 for count in word_freq.values() if count > 1) / len(word_freq)
        features['word_repeat_rate'] = repeat_rate
        
        return features
    
    def comprehensive_analysis(self, text):
        """综合分析"""
        basic_features = TextFeatureExtractor().extract_linguistic_features(text)
        perplexity = self.calculate_perplexity(text)
        writing_pattern = self.analyze_writing_pattern(text)
        semantic_embedding = self.sentence_model.encode([text])[0]
        
        # 合并所有特征
        all_features = {
            **basic_features,
            'perplexity': perplexity,
            **writing_pattern,
            'semantic_embedding': semantic_embedding
        }
        
        return all_features

6. 部署与生产环境配置

6.1 Docker容器化部署

# Dockerfile
FROM python:3.9-slim

WORKDIR /app

# 安装系统依赖
RUN apt-get update && apt-get install -y \
    gcc \
    g++ \
    && rm -rf /var/lib/apt/lists/*

# 复制依赖文件
COPY requirements.txt .

# 安装Python依赖
RUN pip install --no-cache-dir -r requirements.txt

# 复制应用代码
COPY app/ ./app/
COPY config/ ./config/
COPY ai_detector_model.pkl .

# 创建非root用户
RUN useradd -m -u1000 appuser
USER appuser

# 暴露端口
EXPOSE 8000

# 启动命令
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

6.2 生产环境配置

# 文件路径：config/settings.py
import os
from typing import Optional

class Settings:
    # 数据库配置
    DATABASE_URL: str = os.getenv("DATABASE_URL", "postgresql://user:pass@localhost/ai_detection")
    
    # 模型配置
    MODEL_PATH: str = os.getenv("MODEL_PATH", "ai_detector_model.pkl")
    
    # API配置
    API_HOST: str = os.getenv("API_HOST", "0.0.0.0")
    API_PORT: int = int(os.getenv("API_PORT", "8000"))
    
    # 安全配置
    CORS_ORIGINS: list = os.getenv("CORS_ORIGINS", "*").split(",")
    RATE_LIMIT: str = os.getenv("RATE_LIMIT", "100/minute")
    
    # 性能配置
    MAX_CONTENT_LENGTH: int = int(os.getenv("MAX_CONTENT_LENGTH", "10485760"))  # 10MB
    MODEL_CACHE_SIZE: int = int(os.getenv("MODEL_CACHE_SIZE", "1000"))

settings = Settings()

6.3 监控与日志配置

# 文件路径：app/utils/monitoring.py
import logging
import time
from functools import wraps
from prometheus_client import Counter, Histogram, generate_latest

# 指标定义
REQUEST_COUNT = Counter('api_requests_total', 'Total API requests', ['method', 'endpoint'])
REQUEST_DURATION = Histogram('api_request_duration_seconds', 'API request duration')

def setup_logging():
    """配置日志"""
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        handlers=[
            logging.FileHandler('app.log'),
            logging.StreamHandler()
        ]
    )

def monitor_requests(func):
    """请求监控装饰器"""
    @wraps(func)
    async def wrapper(*args, **kwargs):
        start_time = time.time()
        
        # 记录请求
        REQUEST_COUNT.labels(
            method=getattr(args[0], 'method', 'UNKNOWN'),
            endpoint=getattr(args[0], 'path', 'UNKNOWN')
        ).inc()
        
        try:
            response = await func(*args, **kwargs)
            duration = time.time() - start_time
            REQUEST_DURATION.observe(duration)
            return response
        except Exception as e:
            logging.error(f"Request failed: {str(e)}")
            raise
    
    return wrapper

7. 常见问题与解决方案

7.1 性能优化问题

问题1：长文本处理速度慢

原因：BERT模型处理长文本时计算复杂度高
解决方案 ：

def optimize_long_text_processing(text, max_length=512):
    """优化长文本处理"""
    if len(text) > max_length:
        # 分段处理，取平均概率
        segments = [text[i:i+max_length] for i in range(0, len(text), max_length)]
        probabilities = []
        
        for segment in segments:
            features = extract_features(segment)
            prob = detector.predict(features.reshape(1, -1))[0]
            probabilities.append(prob)
        
        return np.mean(probabilities)
    else:
        features = extract_features(text)
        return detector.predict(features.reshape(1, -1))[0]

问题2：模型内存占用过大

解决方案 ：使用模型量化技术

def optimize_model_memory():
    """模型内存优化"""
    import torch
    from torch.quantization import quantize_dynamic
    
    # 动态量化
    model_quantized = quantize_dynamic(
        original_model, {torch.nn.Linear}, dtype=torch.qint8
    )
    return model_quantized

7.2 准确率提升策略

策略1：集成多个检测方法

class EnsembleDetector:
    def __init__(self):
        self.detectors = [
            AIContentDetector(),
            AdvancedAIDetector(),
            # 可以添加更多检测器
        ]
        self.weights = [0.4, 0.6]  # 根据性能调整权重
    
    def predict_ensemble(self, text):
        """集成预测"""
        predictions = []
        
        for detector in self.detectors:
            try:
                if hasattr(detector, 'predict'):
                    prob = detector.predict(text)
                else:
                    features = extract_features(text)
                    prob = detector.predict(features.reshape(1, -1))[0]
                predictions.append(prob)
            except Exception as e:
                predictions.append(0.5)  # 中性概率
        
        # 加权平均
        final_prob = sum(p * w for p, w in zip(predictions, self.weights))
        return final_prob

7.3 错误处理与容错机制

# 文件路径：app/utils/error_handling.py
from functools import wraps
import logging

def robust_detection(max_retries=3):
    """鲁棒性检测装饰器"""
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            last_exception = None
            
            for attempt in range(max_retries):
                try:
                    return func(*args, **kwargs)
                except Exception as e:
                    last_exception = e
                    logging.warning(f"检测失败，第{attempt + 1}次重试: {str(e)}")
                    continue
            
            # 所有重试都失败，返回中性结果
            logging.error(f"检测完全失败，返回默认值: {str(last_exception)}")
            return 0.5  # 中性概率
        
        return wrapper
    return decorator

class FallbackDetector:
    """降级检测器"""
    def __init__(self):
        self.primary_detector = AIContentDetector()
        self.fallback_rules = [
            self.rule_based_detection,
            self.statistical_detection
        ]
    
    def rule_based_detection(self, text):
        """基于规则的检测"""
        # 简单的启发式规则
        ai_indicators = [
            "根据当前", "综上所述", "需要注意的是",
            "首先", "其次", "最后"
        ]
        
        score = 0
        for indicator in ai_indicators:
            if indicator in text:
                score += 0.1
        
        return min(score, 1.0)
    
    def detect_with_fallback(self, text):
        """带降级的检测"""
        try:
            return self.primary_detector.predict(text)
        except Exception as e:
            logging.error(f"主检测器失败，使用降级方案: {str(e)}")
            
            # 尝试降级方案
            for rule in self.fallback_rules:
                try:
                    return rule(text)
                except:
                    continue
            
            return 0.5  # 最终降级

8. 最佳实践与工程建议

8.1 数据质量保障

训练数据收集原则 ：

确保数据来源的多样性和代表性
人工标注数据需要多人交叉验证
定期更新训练数据以适应新的AI模型
建立数据质量监控机制

class DataQualityValidator:
    """数据质量验证器"""
    
    @staticmethod
    def validate_training_data(texts, labels):
        """验证训练数据质量"""
        issues = []
        
        # 检查数据平衡
        ai_count = sum(labels)
        human_count = len(labels) - ai_count
        balance_ratio = min(ai_count, human_count) / max(ai_count, human_count)
        
        if balance_ratio < 0.3:
            issues.append(f"数据不平衡: AI样本{ai_count}, 人工样本{human_count}")
        
        # 检查文本质量
        for i, text in enumerate(texts):
            if len(text.strip()) < 50:
                issues.append(f"文本过短: 索引{i}")
            if len(text.split()) < 10:
                issues.append(f"词汇量过少: 索引{i}")
        
        return issues

8.2 模型更新与维护

持续学习策略 ：

class ContinuousLearning:
    def __init__(self, detector, validation_threshold=0.8):
        self.detector = detector
        self.threshold = validation_threshold
        self.feedback_data = []
    
    def add_feedback(self, text, is_ai, confidence):
        """添加用户反馈"""
        if confidence < self.threshold:
            self.feedback_data.append({
                'text': text,
                'label': 1 if is_ai else 0,
                'confidence': confidence
            })
    
    def retrain_if_needed(self, min_samples=100):
        """根据需要重新训练"""
        if len(self.feedback_data) >= min_samples:
            self._perform_retraining()
    
    def _perform_retraining(self):
        """执行重新训练"""
        # 实现增量学习逻辑
        pass

8.3 安全与隐私考虑

隐私保护措施 ：

文本内容哈希处理，不存储原始内容
实施数据访问控制
定期清理历史数据
遵守相关数据保护法规

class PrivacyProtector:
    """隐私保护器"""
    
    @staticmethod
    def anonymize_text(text, keep_ratio=0.7):
        """文本匿名化处理"""
        words = text.split()
        keep_count = int(len(words) * keep_ratio)
        
        # 保留关键信息，匿名化其他内容
        important_words = words[:keep_count]
        anonymized = important_words + ['[匿名]'] * (len(words) - keep_count)
        
        return ' '.join(anonymized)
    
    @staticmethod
    def should_store_content(text, min_length=100):
        """判断是否需要存储内容"""
        return len(text) >= min_length

通过本文的完整实现，我们建立了一个可扩展的AI内容检测系统。在实际项目中，建议根据具体需求调整阈值和算法参数，并持续优化模型性能。这种技术方案不仅适用于内容标记，还可以扩展到版权保护、内容审核等多个应用场景。