BEV感知+Occupancy网络：3D目标检测与占据栅格预测

一、引言

自动驾驶的核心挑战之一是精确理解周围的三维环境。传统2D检测丢失深度信息、LiDAR昂贵且稀疏。BEV（Bird’s Eye View）感知将多视角相机图像统一变换到鸟瞰视角，实现精确的3D检测。Occupancy网络更进一步，预测每个体素是否被占据，处理任意形状障碍物。

本文将深入 BEVFormer 架构、从相机到BEV的几何变换、以及 Occ3D 占据栅格预测。

二、相机→BEV 几何变换

2.1 IPM（逆透视变换）

import numpy as np
import cv2

def compute_ipm_matrix(K, R, T, H=3, W=50, resolution=0.1):
    """计算逆透视变换矩阵
    K: 内参 [3,3], R: 旋转 [3,3], T: 平移 [3]
    输出: BEV尺寸 H×W，分辨率 resolution 米/像素"""
    
    # 构建外参矩阵
    RT = np.hstack([R, T.reshape(3, 1)])
    
    # 定义地面平面点（在世界坐标系 z=0 平面上）
    grid_x = (np.arange(W) - W // 2) * resolution
    grid_y = (np.arange(H) - H // 2) * resolution
    grid_xx, grid_yy = np.meshgrid(grid_x, grid_y)
    
    # 世界坐标 → 相机坐标
    world_points = np.stack([grid_xx.flatten(),
                              grid_yy.flatten(),
                              np.zeros_like(grid_xx.flatten()),
                              np.ones_like(grid_xx.flatten())], axis=0)
    
    cam_points = RT @ world_points  # [3, N]
    
    # 相机坐标 → 像素坐标
    pixel_points = K @ cam_points
    pixel_points = pixel_points[:2] / pixel_points[2]  # 透视除法
    
    return pixel_points.reshape(2, H, W).transpose(1, 2, 0)

2.2 Lift-Splat-Shoot（BEVDet核心）

class LiftSplatShoot(nn.Module):
    """LSS: 从多视角图像提升到3D，再投影到BEV"""
    
    def __init__(self, frustum_depth=60, grid_size=(200, 200)):
        super().__init__()
        self.D = frustum_depth  # 深度离散化（1-60米）
        self.H, self.W = grid_size
        
        # 深度分布网络（每个像素预测深度概率）
        self.depth_net = nn.Sequential(
            nn.Conv2d(256, 256, 3, padding=1),
            nn.ReLU(),
            nn.Conv2d(256, self.D, 1)  # 输出D类深度分布
        )
        
        # 特征网络
        self.feature_net = nn.Conv2d(256, 64, 1)
    
    def forward(self, images, extrinsics, intrinsics):
        """images: [B,N,3,H,W] N个相机"""
        B, N, C, H, W = images.shape
        
        # 1. Lift: 每个像素 → 3D视锥
        features = self.encode(images)  # [B,N,C',H,W]
        
        # 深度分布
        depth_probs = self.depth_net(features).softmax(dim=1)  # [B*N,D,H,W]
        
        # 特征与深度外积
        pixel_features = self.feature_net(features)  # [B*N,64,H,W]
        # 提升到3D: [B*N,D,H,W] × [B*N,64,H,W] → [B*N,64,D,H,W]
        frustum_features = depth_probs.unsqueeze(1) * pixel_features.unsqueeze(2)
        
        # 2. Splat: 3D视锥 → BEV (相机坐标 → 世界BEV坐标)
        bev_grid = self.compute_bev_grid(extrinsics, intrinsics, B, N)
        
        # 使用视锥点在BEV坐标系的位置，将特征 scatter 到BEV网格
        bev_features = self.splat_to_bev(frustum_features, bev_grid)
        
        return bev_features
    
    def splat_to_bev(self, frustum_features, bev_indices):
        """将3D视锥 splat 到2D BEV平面（求和池化）"""
        B, C, D, H, W = frustum_features.shape
        bev = torch.zeros(B, C, self.H, self.W, device=frustum_features.device)
        
        # 使用scatter_add高效实现
        for b in range(B):
            indices = bev_indices[b]  # [D,H,W,2] → (x_idx, y_idx)
            for d in range(D):
                # 过滤有效坐标
                valid = (indices[d,:,:,0] >= 0) & (indices[d,:,:,0] < self.W) & \
                        (indices[d,:,:,1] >= 0) & (indices[d,:,:,1] < self.H)
                # Sum pooling
                idx = indices[d,valid]
                bev[b].index_put_(
                    (idx[:,1], idx[:,0]),
                    frustum_features[b,:,d][:,valid],
                    accumulate=True
                )
        
        return bev

三、BEVFormer 架构

class BEVFormer(nn.Module):
    """BEVFormer: 使用时空Transformer的BEV感知"""
    
    def __init__(self, bev_h=200, bev_w=200, embed_dim=256):
        super().__init__()
        self.bev_h, self.bev_w = bev_h, bev_w
        
        # BEV查询（可学习的位置编码）
        self.bev_queries = nn.Parameter(
            torch.randn(bev_h * bev_w, embed_dim)
        )
        
        # 空间交叉注意力（Image → BEV）
        self.spatial_cross_attention = SpatialCrossAttention(embed_dim)
        
        # 时间自注意力（融合历史BEV特征）
        self.temporal_self_attention = TemporalSelfAttention(embed_dim)
        
        # BEV编码器层
        self.encoder_layers = nn.ModuleList([
            BEVFormerLayer(embed_dim) for _ in range(6)
        ])
    
    def forward(self, multi_view_features, prev_bev=None):
        """multi_view_features: [B,N,C,H,W]"""
        B = multi_view_features.shape[0]
        
        # 1. 初始化BEV查询
        bev_queries = self.bev_queries.unsqueeze(0).repeat(B, 1, 1)
        
        # 2. 时间自注意力（融合历史BEV）
        if prev_bev is not None:
            bev_queries = self.temporal_self_attention(
                bev_queries, prev_bev
            )
        
        # 3. 空间交叉注意力 + FFN（多层堆叠）
        for layer in self.encoder_layers:
            bev_queries = layer(bev_queries, multi_view_features)
        
        # 4. 重塑为BEV特征图
        bev_features = bev_queries.reshape(B, self.bev_h, self.bev_w, -1)
        return bev_features


class SpatialCrossAttention(nn.Module):
    """空间交叉注意力: BEV查询 → 参考点 → 采样图像特征"""
    
    def __init__(self, embed_dim=256, num_points=4):
        super().__init__()
        self.embed_dim = embed_dim
        self.num_points = num_points
        
        # 可变形注意力
        self.deformable_attention = DeformableAttention(embed_dim)
        
        # 采样偏移预测
        self.sampling_offsets = nn.Linear(embed_dim, num_points * 2)
        self.attention_weights = nn.Linear(embed_dim, num_points)
    
    def forward(self, bev_queries, image_features, reference_points):
        """
        bev_queries: [B, N_bev, C]
        reference_points: [B, N_bev, num_views, 2]（每个BEV点在每个相机视角的投影坐标）
        """
        B, N_bev, C = bev_queries.shape
        N_view = image_features.shape[1]
        
        # 预测采样偏移和注意力权重
        offsets = self.sampling_offsets(bev_queries)  # [B,N_bev,4*2]
        attn_weights = self.attention_weights(bev_queries).softmax(-1)
        
        # 在图像特征上采样
        sampled_features = []
        for v in range(N_view):
            # 获取该视角的参考点
            ref_pts = reference_points[:, :, v]  # [B,N_bev,2]
            
            # 2D grid_sample
            pts = ref_pts.unsqueeze(2) + offsets.reshape(B, N_bev, self.num_points, 2)
            pts = pts * 2 - 1  # 归一化到[-1,1]
            
            feat = F.grid_sample(
                image_features[:, v],  # [B,C,H,W]
                pts,  # [B,N_bev,4,2]
                align_corners=True,
                mode='bilinear'
            )  # [B,C,N_bev,4]
            
            sampled_features.append(feat)
        
        # 加权融合
        sampled = torch.stack(sampled_features, dim=-1)  # [B,C,N_bev,4,N_view]
        weighted = (sampled * attn_weights.unsqueeze(1).unsqueeze(-1)).sum(-1).sum(-1)
        
        return weighted.transpose(1, 2)  # [B,N_bev,C]

四、Occupancy 占据栅格

4.1 Occ3D 数据

# Occupancy标注格式: 每个体素一个标签
# 标签集: {0:free, 1:car, 2:truck, 3:pedestrian, ..., 16:traffic_cone}

class Occ3DDecoder(nn.Module):
    """从BEV特征解码占据栅格"""
    
    def __init__(self, bev_dim=256, occ_size=(200,200,16), num_classes=17):
        super().__init__()
        self.occ_h, self.occ_w, self.occ_z = occ_size
        
        # 高度上采样（BEV → 3D）
        self.height_up = nn.Sequential(
            nn.ConvTranspose3d(bev_dim, 128, (4,1,1), stride=(2,1,1)),
            nn.BatchNorm3d(128),
            nn.ReLU(),
            nn.ConvTranspose3d(128, 64, (4,1,1), stride=(2,1,1)),
        )
        
        # 分割头
        self.seg_head = nn.Sequential(
            nn.Conv3d(64, 64, 3, padding=1),
            nn.BatchNorm3d(64),
            nn.ReLU(),
            nn.Conv3d(64, num_classes, 1)
        )
    
    def forward(self, bev_features):
        """bev_features: [B,C,H,W]"""
        # BEV → 3D: 沿Z轴堆叠
        B, C, H, W = bev_features.shape
        occ = bev_features.unsqueeze(2).repeat(1, 1, self.occ_z, 1, 1)
        
        # 3D卷积上采样
        occ = self.height_up(occ)  # [B,64,64,H,W]
        
        # 占据预测
        occ_logits = self.seg_head(occ)  # [B,17,64,H,W]
        
        return occ_logits

# 损失函数
ce_loss = nn.CrossEntropyLoss(ignore_index=255)  # 忽略未知
lovasz_loss = LovaszSoftmax()  # 处理类别不平衡

loss = ce_loss(pred, gt) + 0.5 * lovasz_loss(pred, gt)

五、端到端3D检测器

class BEV3DDetector(nn.Module):
    """基于BEV的端到端3D目标检测器"""
    
    def __init__(self):
        self.backbone = ResNet50()
        self.neck = FPN([256, 512, 1024, 2048], 256)
        self.view_transform = LiftSplatShoot()
        self.bev_encoder = BEVFormer()
        
        # 检测头（类似CenterPoint）
        self.heatmap_head = nn.Sequential(
            nn.Conv2d(256, 256, 3, padding=1),
            nn.BatchNorm2d(256), nn.ReLU(),
            nn.Conv2d(256, 10, 1)  # 10类
        )
        self.reg_head = nn.Conv2d(256, 8, 1)  # x,y,z,dx,dy,dz,sinθ,cosθ
    
    def forward(self, images, calibs):
        # 图像特征提取
        img_feats = self.backbone(images)
        img_feats = self.neck(img_feats)
        
        # 视角变换
        bev_feats = self.view_transform(img_feats, calibs)
        bev_feats = self.bev_encoder(bev_feats)
        
        # 检测
        heatmap = self.heatmap_head(bev_feats)
        reg = self.reg_head(bev_feats)
        
        return heatmap, reg

# NMS后处理
def decode_boxes(heatmap, reg, threshold=0.3):
    """解码BEV检测结果"""
    # 1. 热力图峰值检测（NMS在BEV空间）
    peaks = nms_2d(heatmap, kernel=3)
    
    # 2. 提取检测框
    boxes = []
    for peak in peaks:
        cx, cy = peak
        # 回归参数
        dx, dy, z, length, width, height, sin_y, cos_y = reg[:, cx, cy]
        rot_y = torch.atan2(sin_y, cos_y)
        boxes.append({
            "center": (cx + dx, cy + dy, z),
            "dims": (length, width, height),
            "yaw": rot_y,
            "score": heatmap[:, cx, cy].max()
        })
    
    return boxes

六、数据增强与训练技巧

# BEV专用数据增强
class BEVAugmentation:
    @staticmethod
    def random_flip(images, boxes, calibs, p=0.5):
        """左右翻转"""
        if random.random() < p:
            images = torch.flip(images, dims=[-1])
            boxes[:, 0] = -boxes[:, 0]
            boxes[:, 6] = -boxes[:, 6]  # yaw
        return images, boxes, calibs
    
    @staticmethod
    def random_rotate(bev_feats, boxes, angle_range=(-22.5, 22.5)):
        """BEV空间随机旋转"""
        angle = random.uniform(*angle_range)
        # 旋转BEV特征图
        M = cv2.getRotationMatrix2D((W/2, H/2), angle, 1.0)
        bev_feats = cv2.warpAffine(bev_feats, M, (W, H))
        # 旋转检测框
        boxes[:, 6] += np.radians(angle)
        return bev_feats, boxes

七、主流框架对比

八、总结

BEV感知核心要点：

1. 视角变换是关键 — LSS的Lift-Splat-Shoot是工程核心
2. Transformer是标配 — 空间交叉注意力 + 时间自注意力
3. Occupancy是未来 — 从3D框到稠密占据栅格，处理任意形状障碍物
4. 时序融合提升大 — 单帧→多帧，mAP提升5-10个点