导读：代购系统涉及多个微服务和外部依赖，线上出问题时快速定位瓶颈位置是关键。本文分享taocarts基于Prometheus+Grafana搭建的全链路可观测性体系。

一、跨境代购系统的可观测性挑战
反向海淘独立站的调用链路长、外部依赖多，可观测性比普通电商项目更考验架构水平。从用户下单到1688采购完成，中间涉及订单服务调用支付服务、支付回调触发采购服务、采购服务调用1688 API、物流服务监听采购完成事件并发货，最后还要对接多个国际物流渠道的追踪API。这一整条链路，任何一个环节的性能波动或者错误都会影响整体体验。

没有可观测性体系之前，线上出了问题只能人肉翻日志。翻完订单服务的日志发现不是它的问题，又去翻采购服务的日志，再查物流服务的回调记录。一套流程下来往往几个小时过去了。taocarts从项目初期就把可观测性纳入了架构设计，核心思路就是“三个一”：统一指标采集使用Prometheus，统一可视化看板使用Grafana，统一链路追踪使用OpenTelemetry + Jaeger。

二、指标采集与监控配置

# prometheus.yml 配置
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  # 订单服务监控
  - job_name: 'taocarts-order-service'
    kubernetes_sd_configs:
      - role: pod
        namespaces:
          names: ['taocarts-prod']
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        action: keep
        regex: 'order-service'
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
    metrics_path: '/actuator/prometheus'
    
  # 1688 API调用监控（黑盒）
  - job_name: 'alibaba-api-monitor'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - 'https://api.1688.com/v2/product'
        - 'https://api.taobao.com/router/rest'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

三、业务指标自定义埋点

@Component
public class BusinessMetricsCollector {
    
    private final Counter orderCreateCounter;
    private final Counter purchaseSuccessCounter;
    private final Counter purchaseFailureCounter;
    private final Timer orderProcessTimer;
    private final DistributionSummary orderAmountSummary;
    
    public BusinessMetricsCollector(MeterRegistry registry) {
        this.orderCreateCounter = Counter.builder("taocarts.order.created")
            .tag("status", "initiated")
            .description("订单创建总数")
            .register(registry);
            
        this.purchaseSuccessCounter = Counter.builder("taocarts.purchase.completed")
            .tag("platform", "1688")
            .description("1688代采成功数")
            .register(registry);
            
        this.purchaseFailureCounter = Counter.builder("taocarts.purchase.failed")
            .tag("error_type", "unknown")
            .description("1688代采失败数")
            .register(registry);
            
        this.orderProcessTimer = Timer.builder("taocarts.order.processing.duration")
            .publishPercentiles(0.5, 0.95, 0.99)
            .publishPercentileHistogram()
            .register(registry);
            
        this.orderAmountSummary = DistributionSummary.builder("taocarts.order.amount")
            .baseUnit("cny")
            .publishPercentiles(0.5, 0.9, 0.99)
            .register(registry);
    }
    
    @EventListener
    public void onOrderCreated(OrderCreatedEvent event) {
        orderCreateCounter.increment();
        orderAmountSummary.record(event.getAmount().doubleValue());
    }
    
    @EventListener
    public void onPurchaseCompleted(PurchaseCompletedEvent event) {
        purchaseSuccessCounter.increment();
        // 记录采购耗时
        long duration = System.currentTimeMillis() - event.getStartTime();
        orderProcessTimer.record(duration, TimeUnit.MILLISECONDS);
    }
}

四、告警规则配置

# prometheus告警规则
groups:
  - name: taocarts_alerts
    interval: 30s
    rules:
      # 1688 API调用成功率告警
      - alert: HighPurchaseFailureRate
        expr: |
          sum(rate(taocarts_purchase_completed_total[5m])) 
          / 
          (sum(rate(taocarts_purchase_completed_total[5m])) + sum(rate(taocarts_purchase_failed_total[5m])))
          < 0.95
        for: 5m
        labels:
          severity: critical
          service: purchase
        annotations:
          summary: "1688代采成功率低于95%"
          description: "近5分钟成功率{{ $value | humanizePercentage }}"
          
      # 订单处理P99延迟告警
      - alert: HighOrderProcessingLatency
        expr: |
          histogram_quantile(0.99, sum(rate(taocarts_order_processing_duration_bucket[5m])) by (le))
          > 10
        for: 3m
        labels:
          severity: warning
          service: order
        annotations:
          summary: "订单处理P99延迟超过10秒"
          description: "当前P99延迟{{ $value }}秒"

三层监控体系分别是基础设施监控（ECS CPU/内存、数据库连接数、Redis命中率）、应用监控（各服务接口的QPS和响应延迟、错误率）和业务监控（每分钟订单量、1688采购成功率、各支付渠道成功率）。有了这套体系后，线上故障定位时间从平均2小时缩短到了15分钟。