示例

import numpy as np
import pandas as pd
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score


# ==================== 定义函数 ====================
def ignore_low_variance(X_train, X_test, threshold=0.01, use_std=False,
                        scale_data=False, verbose=True):
    """
    删除低方差列

    Parameters:
    -----------
    X_train, X_test : DataFrame
        训练集和测试集
    threshold : float
        方差阈值
    use_std : bool
        是否使用标准差模式
    scale_data : bool
        是否先标准化数据（推荐）
    verbose : bool
        是否打印删除信息
    """
    # 1. 只选择数值列
    numeric_cols = X_train.select_dtypes(include=[np.number]).columns
    if len(numeric_cols) < X_train.shape[1]:
        print(f"警告：排除了 {X_train.shape[1] - len(numeric_cols)} 个非数值列")

    X_train_num = X_train[numeric_cols].copy()
    X_test_num = X_test[numeric_cols].copy()

    # 2. 可选：标准化数据
    if scale_data:
        from sklearn.preprocessing import StandardScaler
        scaler = StandardScaler()
        X_train_num = pd.DataFrame(
            scaler.fit_transform(X_train_num),
            columns=X_train_num.columns,
            index=X_train_num.index
        )
        X_test_num = pd.DataFrame(
            scaler.transform(X_test_num),
            columns=X_test_num.columns,
            index=X_test_num.index
        )

    # 3. 方差过滤
    if use_std:
        threshold = threshold ** 2

    selector = VarianceThreshold(threshold=threshold)
    X_train_transformed = selector.fit_transform(X_train_num)
    X_test_transformed = selector.transform(X_test_num)

    # 4. 获取保留的列
    cols_kept = X_train_num.columns[selector.get_support()].tolist()
    cols_removed = X_train_num.columns[~selector.get_support()].tolist()

    if verbose and cols_removed:
        print(f"删除了 {len(cols_removed)} 个低方差列: {cols_removed}")

    # 5. 返回DataFrame，包含所有原始列（包括非数值列）
    if len(numeric_cols) < X_train.shape[1]:
        # 保留非数值列
        non_numeric_cols = X_train.columns.difference(numeric_cols)
        result_train = pd.concat([
            pd.DataFrame(X_train_transformed, columns=cols_kept, index=X_train.index),
            X_train[non_numeric_cols]
        ], axis=1)
        result_test = pd.concat([
            pd.DataFrame(X_test_transformed, columns=cols_kept, index=X_test.index),
            X_test[non_numeric_cols]
        ], axis=1)
        return result_train, result_test
    else:
        return (pd.DataFrame(X_train_transformed, columns=cols_kept, index=X_train.index),
                pd.DataFrame(X_test_transformed, columns=cols_kept, index=X_test.index))

调用示例

print("=" * 60)
print("示例1：模拟数据 - 比较不同策略的效果")
print("=" * 60)

# 创建包含不同方差特征的模拟数据
np.random.seed(42)
n_samples = 500

# 创建不同类型的特征
data = {
    'feature_high_var': np.random.randn(n_samples) * 10,  # 高方差
    'feature_medium_var': np.random.randn(n_samples) * 2,  # 中方差
    'feature_low_var': np.random.randn(n_samples) * 0.1,  # 低方差
    'feature_near_constant': np.random.randn(n_samples) * 0.001 + 5,  # 近常数
    'feature_binary': np.random.binomial(1, 0.5, n_samples),  # 二值特征
    'feature_constant': np.ones(n_samples) * 3,  # 常数特征
}

X = pd.DataFrame(data)
y = (X['feature_high_var'] + X['feature_binary'] * 2 + np.random.randn(n_samples) * 0.5 > 0).astype(int)

# 划分数据集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

print("原始数据形状:", X_train.shape)
print("\n各特征方差:")
print(X_train.var().round(6))

# 策略1：不标准化，使用默认阈值
print("\n" + "-" * 40)
print("策略1：不标准化，阈值=0.01")
X_train1, X_test1 = ignore_low_variance(X_train, X_test, threshold=0.01, scale_data=False)
print(f"处理后形状: {X_train1.shape}")

# 策略2：先标准化再删除（推荐）
print("\n" + "-" * 40)
print("策略2：先标准化，阈值=0.01（推荐）")
X_train2, X_test2 = ignore_low_variance(X_train, X_test, threshold=0.01, scale_data=True)
print(f"处理后形状: {X_train2.shape}")

# 策略3：使用标准差模式
print("\n" + "-" * 40)
print("策略3：使用标准差模式，阈值=0.1")
X_train3, X_test3 = ignore_low_variance(X_train, X_test, threshold=0.1, use_std=True, scale_data=True)
print(f"处理后形状: {X_train3.shape}")

输出

原始数据形状: (350, 6)

各特征方差:
feature_high_var         98.129830
feature_medium_var        3.586963
feature_low_var           0.010559
feature_near_constant     0.000001
feature_binary            0.250053
feature_constant          0.000000
dtype: float64

----------------------------------------
策略1：不标准化，阈值=0.01
删除了 2 个低方差列: ['feature_near_constant', 'feature_constant']
处理后形状: (350, 4)

----------------------------------------
策略2：先标准化，阈值=0.01（推荐）
删除了 1 个低方差列: ['feature_constant']
处理后形状: (350, 5)

----------------------------------------
策略3：使用标准差模式，阈值=0.1
删除了 1 个低方差列: ['feature_constant']
处理后形状: (350, 5)

说明： 方法针对的是数值列，一般数据先标准化 再移除低方差列