1. 为什么需要实时监控训练曲线？

在深度学习模型训练过程中，我们经常会遇到这样的困扰：盯着终端里不断跳动的数字，却难以直观感受模型的学习状态。等到训练结束后查看loss和acc曲线，才发现模型早在第10个epoch就过拟合了，白白浪费了后面90个epoch的计算资源。这种"黑盒"式的训练方式，就像蒙着眼睛开车，既低效又危险。

实时可视化训练曲线能带来三个核心优势：

• 即时发现问题 ：当loss曲线突然飙升或准确率停滞不前时，可以立即暂停训练调整超参数
• 节省计算成本 ：观察到模型收敛后及时终止训练，避免无意义的迭代
• 直观理解训练动态 ：通过曲线波动判断学习率是否合适、batch size是否需要调整

我在实际项目中就遇到过这样的情况：训练ResNet分类模型时，前30个epoch验证准确率稳步提升，但突然从第31个epoch开始断崖式下跌。幸亏实时监控发现了这个异常，及时保存了第30个epoch的模型权重，否则整个训练就前功尽弃了。

2. 基础实现方案

2.1 传统的事后绘图方法

最常见的绘图方式是训练完成后读取日志文件绘制曲线。假设我们使用PyTorch框架，典型实现如下：

import matplotlib.pyplot as plt

def save_log(epoch, train_loss, val_loss, train_acc, val_acc):
    with open('training_log.txt', 'a') as f:
        f.write(f"{epoch},{train_loss},{val_loss},{train_acc},{val_acc}\n")

def plot_curves():
    epochs, t_loss, v_loss, t_acc, v_acc = [], [], [], [], []
    with open('training_log.txt', 'r') as f:
        for line in f:
            e, tl, vl, ta, va = map(float, line.strip().split(','))
            epochs.append(e)
            t_loss.append(tl)
            v_loss.append(vl)
            t_acc.append(ta)
            v_acc.append(va)
    
    plt.figure(figsize=(12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(epochs, t_loss, label='Train')
    plt.plot(epochs, v_loss, label='Validation')
    plt.title('Loss Curve')
    plt.legend()
    
    plt.subplot(1, 2, 2)
    plt.plot(epochs, t_acc, label='Train')
    plt.plot(epochs, v_acc, label='Validation') 
    plt.title('Accuracy Curve')
    plt.legend()
    plt.savefig('training_curves.png')

这种方法虽然简单，但存在明显缺陷：

1. 无法实时观察训练状态
2. 当训练意外中断时会丢失所有日志
3. 需要额外处理文件读写，代码不够优雅

2.2 实时可视化的核心思路

实现动态更新的关键在于：

1. 交互式绘图模式 ：使用 plt.ion() 开启交互模式
2. 增量式更新 ：每次epoch只更新曲线数据而非重新绘制
3. 内存数据存储 ：将指标保存在内存列表而非文件中

基础代码框架如下：

import matplotlib.pyplot as plt

plt.ion()  # 开启交互模式
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))

# 初始化曲线
train_line, = ax1.plot([], [], 'r-', label='Train')
val_line, = ax2.plot([], [], 'b-', label='Validation')

def update_plot(epoch, train_loss, val_loss):
    train_line.set_xdata(list(range(epoch+1)))
    train_line.set_ydata(train_losses)
    val_line.set_xdata(list(range(epoch+1)))
    val_line.set_ydata(val_losses)
    fig.canvas.draw()
    fig.canvas.flush_events()

3. 完整实现方案

3.1 Jupyter Notebook中的实现

在Jupyter环境中实现动态可视化最为方便，因为可以直接在单元格输出图像。我们创建一个训练监控类：

class TrainingMonitor:
    def __init__(self):
        self.fig, (self.ax1, self.ax2) = plt.subplots(1, 2, figsize=(12, 5))
        self.train_loss_line, = self.ax1.plot([], [], 'r-', label='Train')
        self.val_loss_line, = self.ax1.plot([], [], 'b-', label='Validation')
        self.train_acc_line, = self.ax2.plot([], [], 'g-', label='Train')
        self.val_acc_line, = self.ax2.plot([], [], 'm-', label='Validation')
        
        # 初始化图表
        self.ax1.set_title('Loss Curve')
        self.ax1.set_xlabel('Epoch')
        self.ax1.set_ylabel('Loss')
        self.ax1.legend()
        
        self.ax2.set_title('Accuracy Curve') 
        self.ax2.set_xlabel('Epoch')
        self.ax2.set_ylabel('Accuracy')
        self.ax2.legend()
        
        self.fig.tight_layout()
    
    def update(self, epoch, train_loss, val_loss, train_acc, val_acc):
        # 更新曲线数据
        x = list(range(epoch+1))
        self.train_loss_line.set_xdata(x)
        self.train_loss_line.set_ydata(train_loss)
        self.val_loss_line.set_xdata(x)
        self.val_loss_line.set_ydata(val_loss)
        
        self.train_acc_line.set_xdata(x)
        self.train_acc_line.set_ydata(train_acc)
        self.val_acc_line.set_xdata(x)
        self.val_acc_line.set_ydata(val_acc)
        
        # 调整坐标轴范围
        self.ax1.relim()
        self.ax1.autoscale_view()
        self.ax2.relim()
        self.ax2.autoscale_view()
        
        # 重绘
        self.fig.canvas.draw()

使用时只需在训练循环中调用update方法：

monitor = TrainingMonitor()

for epoch in range(epochs):
    # 训练代码...
    train_loss = ...
    val_loss = ...
    train_acc = ...
    val_acc = ...
    
    monitor.update(epoch, train_loss, val_loss, train_acc, val_acc)

3.2 终端环境中的实现

在纯终端环境中，我们需要解决两个问题：

1. 防止图像窗口阻塞训练进程
2. 处理Matplotlib的后端兼容性

改进后的实现方案：

import matplotlib
matplotlib.use('Agg')  # 使用非交互式后端
import matplotlib.pyplot as plt
from IPython.display import clear_output

class TerminalMonitor:
    def __init__(self):
        self.fig, (self.ax1, self.ax2) = plt.subplots(1, 2, figsize=(12, 5))
        self.history = {
            'train_loss': [],
            'val_loss': [],
            'train_acc': [],
            'val_acc': []
        }
    
    def update(self, epoch, train_loss, val_loss, train_acc, val_acc):
        self.history['train_loss'].append(train_loss)
        self.history['val_loss'].append(val_loss)
        self.history['train_acc'].append(train_acc)
        self.history['val_acc'].append(val_acc)
        
        # 每5个epoch更新一次图像
        if epoch % 5 == 0:
            self._plot()
    
    def _plot(self):
        clear_output(wait=True)  # 清除之前的输出
        
        epochs = range(len(self.history['train_loss']))
        
        self.ax1.clear()
        self.ax1.plot(epochs, self.history['train_loss'], 'r-', label='Train')
        self.ax1.plot(epochs, self.history['val_loss'], 'b-', label='Validation')
        self.ax1.set_title('Loss Curve')
        self.ax1.legend()
        
        self.ax2.clear()
        self.ax2.plot(epochs, self.history['train_acc'], 'g-', label='Train')
        self.ax2.plot(epochs, self.history['val_acc'], 'm-', label='Validation')
        self.ax2.set_title('Accuracy Curve')
        self.ax2.legend()
        
        plt.tight_layout()
        plt.show()

4. 高级技巧与优化

4.1 平滑处理波动曲线

训练曲线常会出现剧烈波动，影响判断。我们可以使用指数移动平均（EMA）进行平滑：

class SmoothMonitor(TrainingMonitor):
    def __init__(self, alpha=0.2):
        super().__init__()
        self.alpha = alpha
        self.smooth_history = {
            'train_loss': [],
            'val_loss': [],
            'train_acc': [],
            'val_acc': []
        }
    
    def _ema(self, new_val, last_val):
        return self.alpha * new_val + (1 - self.alpha) * last_val
    
    def update(self, epoch, train_loss, val_loss, train_acc, val_acc):
        # 首次更新
        if epoch == 0:
            self.smooth_history['train_loss'].append(train_loss)
            self.smooth_history['val_loss'].append(val_loss)
            self.smooth_history['train_acc'].append(train_acc)
            self.smooth_history['val_acc'].append(val_acc)
        else:
            self.smooth_history['train_loss'].append(
                self._ema(train_loss, self.smooth_history['train_loss'][-1]))
            self.smooth_history['val_loss'].append(
                self._ema(val_loss, self.smooth_history['val_loss'][-1]))
            self.smooth_history['train_acc'].append(
                self._ema(train_acc, self.smooth_history['train_acc'][-1]))
            self.smooth_history['val_acc'].append(
                self._ema(val_acc, self.smooth_history['val_acc'][-1]))
        
        super().update(epoch, 
                      self.smooth_history['train_loss'],
                      self.smooth_history['val_loss'],
                      self.smooth_history['train_acc'],
                      self.smooth_history['val_acc'])

4.2 多GPU训练支持

在多GPU训练时，我们需要聚合各个进程的指标。以PyTorch的DistributedDataParallel为例：

import torch.distributed as dist

def reduce_value(value, average=True):
    world_size = dist.get_world_size()
    if world_size < 2:  # 单GPU情况
        return value
    
    with torch.no_grad():
        dist.all_reduce(value)
        if average:
            value /= world_size
    return value

class DistributedMonitor(TrainingMonitor):
    def update(self, epoch, train_loss, val_loss, train_acc, val_acc):
        # 将指标转换为Tensor
        train_loss = torch.tensor(train_loss).cuda()
        val_loss = torch.tensor(val_loss).cuda()
        train_acc = torch.tensor(train_acc).cuda()
        val_acc = torch.tensor(val_acc).cuda()
        
        # 聚合所有GPU的指标
        train_loss = reduce_value(train_loss)
        val_loss = reduce_value(val_loss)
        train_acc = reduce_value(train_acc)
        val_acc = reduce_value(val_acc)
        
        if dist.get_rank() == 0:  # 只在主进程更新图像
            super().update(epoch, 
                         train_loss.item(),
                         val_loss.item(),
                         train_acc.item(),
                         val_acc.item())

4.3 自动保存最佳模型

结合监控结果实现智能保存：

class SmartCheckpoint(TrainingMonitor):
    def __init__(self, model, patience=5):
        super().__init__()
        self.model = model
        self.patience = patience
        self.best_acc = 0.0
        self.counter = 0
    
    def update(self, epoch, train_loss, val_loss, train_acc, val_acc):
        super().update(epoch, train_loss, val_loss, train_acc, val_acc)
        
        # 保存最佳模型
        if val_acc > self.best_acc:
            self.best_acc = val_acc
            torch.save(self.model.state_dict(), f'best_model_epoch{epoch}.pth')
            self.counter = 0
        else:
            self.counter += 1
        
        # 早停机制
        if self.counter >= self.patience:
            print(f'Early stopping at epoch {epoch}')
            return True  # 返回True表示应该停止训练
        return False

5. 实际应用案例

5.1 图像分类任务监控

在ResNet训练CIFAR-10时，我们观察到：

• 学习率设置过高时，loss曲线会出现剧烈震荡
• batch size过小时，准确率曲线上升缓慢
• 当验证loss开始上升而训练loss继续下降时，就是过拟合的信号

一个典型的健康训练曲线应该呈现以下特征：

1. 训练loss平稳下降，最终趋近于0
2. 验证loss先下降后趋于平稳
3. 训练和验证准确率同步上升，最终保持较小差距

5.2 目标检测任务的特殊处理

目标检测通常有多个loss组件（分类loss、回归loss等），建议分别监控：

class DetectionMonitor:
    def __init__(self):
        self.fig, axes = plt.subplots(2, 2, figsize=(12, 10))
        self.axes = {
            'cls_loss': axes[0, 0],
            'reg_loss': axes[0, 1],
            'total_loss': axes[1, 0],
            'mAP': axes[1, 1]
        }
        self.lines = {}
        
        for name, ax in self.axes.items():
            ax.set_title(name.replace('_', ' ').title())
            ax.set_xlabel('Epoch')
            ax.set_ylabel(name)
            self.lines[name] = ax.plot([], [])[0]
    
    def update(self, epoch, metrics):
        for name, value in metrics.items():
            x_data = list(range(epoch+1))
            y_data = metrics[name]
            self.lines[name].set_data(x_data, y_data)
            self.axes[name].relim()
            self.axes[name].autoscale_view()
        
        plt.tight_layout()
        plt.draw()
        plt.pause(0.01)

5.3 自然语言处理任务的监控技巧

在训练Transformer模型时，特别需要注意：

1. 使用对数坐标显示loss，因为初始loss可能非常大
2. 监控梯度范数防止梯度爆炸
3. 记录并显示学习率变化

class NLPMonitor(TrainingMonitor):
    def __init__(self):
        super().__init__()
        self.ax1.set_yscale('log')  # loss使用对数坐标
        
        # 添加学习率曲线
        self.lr_ax = self.ax1.twinx()
        self.lr_line, = self.lr_ax.plot([], [], 'k:', label='Learning Rate')
        self.ax1.legend(loc='upper left')
        self.lr_ax.legend(loc='upper right')
    
    def update(self, epoch, train_loss, val_loss, train_acc, val_acc, lr):
        super().update(epoch, train_loss, val_loss, train_acc, val_acc)
        
        # 更新学习率曲线
        x = list(range(epoch+1))
        self.lr_line.set_xdata(x)
        self.lr_line.set_ydata([lr]*(epoch+1))
        self.lr_ax.relim()
        self.lr_ax.autoscale_view()

6. 常见问题排查

6.1 图像不更新或闪烁

可能原因及解决方案：

1. 未开启交互模式 ：确保调用了 plt.ion()
2. 未正确处理图形事件 ：在更新后调用 fig.canvas.flush_events()
3. 后端兼容性问题 ：尝试更换后端 matplotlib.use('TkAgg')

6.2 内存泄漏问题

长期运行的监控工具可能出现内存泄漏，解决方法：

1. 定期清理图形对象
2. 使用 gc.collect() 手动触发垃圾回收
3. 避免在循环中重复创建Figure对象

优化后的安全更新方法：

def safe_update(fig, lines, new_data):
    for line, data in zip(lines, new_data):
        line.set_data(data)
    
    fig.canvas.draw()
    fig.canvas.flush_events()
    
    # 手动释放内存
    fig.clf()
    plt.close('all')

6.3 远程服务器显示问题

通过SSH连接服务器时，可以这样解决显示问题：

1. 使用X11转发： ssh -X user@server
2. 或者保存图像到文件定期查看：

if epoch % 10 == 0:
    plt.savefig(f'epoch_{epoch}.png')

7. 性能优化建议

7.1 降低更新频率

对于长时间训练，没必要每个epoch都更新图像：

update_interval = max(1, total_epochs // 100)  # 总共更新100次左右

if epoch % update_interval == 0 or epoch == total_epochs - 1:
    monitor.update(epoch, ...)

7.2 轻量级可视化方案

如果Matplotlib性能不足，可以考虑：

1. 终端绘图 ：使用 tqdm + rich 库在终端显示简易曲线
2. Web可视化 ：将指标发送到TensorBoard或Weights & Biases

终端曲线示例代码：

from tqdm import tqdm
import numpy as np

def plot_terminal(values, height=20):
    min_val = min(values)
    max_val = max(values)
    scale = height / (max_val - min_val + 1e-8)
    
    for v in values:
        bar = '■' * int((v - min_val) * scale)
        print(f"{v:.4f} | {bar}")

7.3 异步更新机制

使用多线程避免绘图阻塞训练：

from threading import Thread
from queue import Queue

class AsyncMonitor:
    def __init__(self):
        self.queue = Queue()
        self.thread = Thread(target=self._update_loop, daemon=True)
        self.thread.start()
    
    def update(self, *args):
        self.queue.put(args)
    
    def _update_loop(self):
        while True:
            data = self.queue.get()
            # 实际更新逻辑
            ...

8. 工程化实践

8.1 封装为回调函数

与主流框架集成的最佳实践：

class VisCallback:
    def __init__(self, model):
        self.model = model
        self.monitor = TrainingMonitor()
    
    def on_epoch_end(self, epoch, logs):
        self.monitor.update(
            epoch,
            logs['train_loss'],
            logs['val_loss'],
            logs['train_acc'],
            logs['val_acc']
        )
        
        # 自动保存最佳模型
        if logs['val_acc'] > self.best_acc:
            self.best_acc = logs['val_acc']
            torch.save(self.model.state_dict(), 'best_model.pth')

# 在训练循环中使用
callback = VisCallback(model)
for epoch in range(epochs):
    # ...训练代码...
    logs = {
        'train_loss': train_loss,
        'val_loss': val_loss,
        'train_acc': train_acc,
        'val_acc': val_acc
    }
    callback.on_epoch_end(epoch, logs)

8.2 与现有框架集成

以PyTorch Lightning为例：

import pytorch_lightning as pl

class VisCallback(pl.Callback):
    def __init__(self):
        self.monitor = TrainingMonitor()
    
    def on_train_epoch_end(self, trainer, module):
        logs = trainer.callback_metrics
        self.monitor.update(
            trainer.current_epoch,
            logs['train_loss'],
            logs['val_loss'],
            logs['train_acc'],
            logs['val_acc']
        )

# 使用方式
trainer = pl.Trainer(callbacks=[VisCallback()])
trainer.fit(model)

8.3 日志与可视化结合

完整的训练监控系统应该包含：

1. 实时可视化曲线
2. 结构化日志存储
3. 关键指标预警机制

class TrainingDashboard:
    def __init__(self):
        self.visualizer = TrainingMonitor()
        self.logger = {
            'train_loss': [],
            'val_loss': [],
            'train_acc': [],
            'val_acc': [],
            'lr': [],
            'timestamp': []
        }
    
    def update(self, epoch, metrics):
        # 更新可视化
        self.visualizer.update(epoch, **metrics)
        
        # 记录日志
        for k, v in metrics.items():
            self.logger[k].append(v)
        self.logger['timestamp'].append(time.time())
        
        # 检查异常
        self._check_anomaly(epoch, metrics)
    
    def _check_anomaly(self, epoch, metrics):
        if len(self.logger['val_loss']) < 2:
            return
        
        # 检测loss突增
        if metrics['val_loss'] > 2 * self.logger['val_loss'][-2]:
            print(f"Warning: val_loss doubled at epoch {epoch}")
        
        # 检测准确率下降
        if metrics['val_acc'] < 0.5 * self.logger['val_acc'][-2]:
            print(f"Warning: val_acc dropped 50% at epoch {epoch}")
    
    def save_logs(self, path):
        pd.DataFrame(self.logger).to_csv(path, index=False)

9. 可视化增强技巧

9.1 动态标注关键点

在曲线上标注关键事件：

def mark_events(ax, events):
    for epoch, label in events.items():
        y = ax.lines[0].get_ydata()[epoch]
        ax.annotate(label, xy=(epoch, y), xytext=(10, 10),
                   textcoords='offset points', arrowprops=dict(arrowstyle='->'))

9.2 多视图协同分析

同时显示学习率、batch size等超参数变化：

class HyperParamVisualizer:
    def __init__(self):
        self.fig, axes = plt.subplots(3, 1, figsize=(10, 12))
        self.axes = {
            'loss': axes[0],
            'acc': axes[1],
            'params': axes[2]
        }
        
        # 初始化曲线
        self.lines = {}
        for name in ['train_loss', 'val_loss']:
            self.lines[name], = self.axes['loss'].plot([], [], label=name)
        
        for name in ['train_acc', 'val_acc']:
            self.lines[name], = self.axes['acc'].plot([], [], label=name)
        
        for name in ['lr', 'batch_size']:
            self.lines[name], = self.axes['params'].plot([], [], label=name)
        
        # 配置图表
        self.axes['loss'].set_title('Loss Curves')
        self.axes['acc'].set_title('Accuracy Curves')
        self.axes['params'].set_title('Hyper Parameters')
        
        for ax in self.axes.values():
            ax.legend()
            ax.grid(True)
    
    def update(self, epoch, metrics):
        x = list(range(epoch+1))
        
        # 更新loss曲线
        self.lines['train_loss'].set_data(x, metrics['train_loss'])
        self.lines['val_loss'].set_data(x, metrics['val_loss'])
        self.axes['loss'].relim()
        self.axes['loss'].autoscale_view()
        
        # 更新accuracy曲线
        self.lines['train_acc'].set_data(x, metrics['train_acc'])
        self.lines['val_acc'].set_data(x, metrics['val_acc'])
        self.axes['acc'].relim()
        self.axes['acc'].autoscale_view()
        
        # 更新超参数曲线
        self.lines['lr'].set_data(x, metrics['lr_history'])
        self.lines['batch_size'].set_data(x, metrics['bs_history'])
        self.axes['params'].relim()
        self.axes['params'].autoscale_view()
        
        self.fig.canvas.draw()

9.3 交互式探索

使用mplcursors库添加交互功能：

import mplcursors

def add_interactive(fig):
    cursor = mplcursors.cursor(hover=True)
    
    @cursor.connect("add")
    def on_add(sel):
        x, y = sel.target
        sel.annotation.set_text(f"Epoch: {int(x)}\nValue: {y:.4f}")
        sel.annotation.get_bbox_patch().set(fc="white", alpha=0.9)
    
    return cursor

10. 延伸应用场景

10.1 模型对比分析

同时显示多个模型的训练曲线：

class ModelComparator:
    def __init__(self, model_names):
        self.fig, (self.ax1, self.ax2) = plt.subplots(1, 2, figsize=(14, 6))
        self.lines = {}
        colors = ['r', 'g', 'b', 'c', 'm', 'y']
        
        for i, name in enumerate(model_names):
            self.lines[f'{name}_loss'], = self.ax1.plot(
                [], [], f'{colors[i]}-', label=f'{name} Loss')
            self.lines[f'{name}_acc'], = self.ax2.plot(
                [], [], f'{colors[i]}--', label=f'{name} Acc')
        
        self.ax1.set_title('Loss Comparison')
        self.ax2.set_title('Accuracy Comparison')
        self.ax1.legend()
        self.ax2.legend()
    
    def update(self, epoch, model_metrics):
        for name, metrics in model_metrics.items():
            x = list(range(epoch+1))
            
            self.lines[f'{name}_loss'].set_data(x, metrics['val_loss'])
            self.lines[f'{name}_acc'].set_data(x, metrics['val_acc'])
        
        self.ax1.relim()
        self.ax1.autoscale_view()
        self.ax2.relim()
        self.ax2.autoscale_view()
        
        self.fig.canvas.draw()

10.2 超参数搜索可视化

配合超参数搜索工具显示不同配置的效果：

class HparamVisualizer:
    def __init__(self):
        self.fig = plt.figure(figsize=(14, 8))
        self.ax1 = self.fig.add_subplot(221)
        self.ax2 = self.fig.add_subplot(222)
        self.ax3 = self.fig.add_subplot(212)
        
        self.config_lines = {}
    
    def add_config(self, config_id, config):
        color = np.random.rand(3,)
        self.config_lines[config_id] = {
            'loss_line': self.ax1.plot([], [], '-', color=color, label=config_id)[0],
            'acc_line': self.ax2.plot([], [], '-', color=color)[0],
            'config': config
        }
        self.ax1.legend()
    
    def update(self, config_id, epoch, metrics):
        lines = self.config_lines[config_id]
        x = list(range(epoch+1))
        
        lines['loss_line'].set_data(x, metrics['val_loss'])
        lines['acc_line'].set_data(x, metrics['val_acc'])
        
        self.ax1.relim()
        self.ax1.autoscale_view()
        self.ax2.relim()
        self.ax2.autoscale_view()
        
        # 在下方显示当前最佳配置
        best_id = max(self.config_lines.keys(),
                     key=lambda k: max(self.config_lines[k]['acc_line'].get_ydata() or [0]))
        best_config = self.config_lines[best_id]['config']
        
        self.ax3.clear()
        self.ax3.axis('off')
        self.ax3.text(0, 0.5, f"Best Config: {best_id}\n{best_config}",
                     fontsize=10, family='monospace')
        
        self.fig.canvas.draw()

10.3 分布式训练监控

监控多节点训练时的设备状态：

class ClusterMonitor:
    def __init__(self, node_names):
        self.fig, axes = plt.subplots(len(node_names), 2, figsize=(12, 3*len(node_names)))
        self.lines = {}
        
        for i, name in enumerate(node_names):
            axes[i, 0].set_title(f'{name} - GPU Util')
            axes[i, 1].set_title(f'{name} - Memory')
            
            self.lines[f'{name}_gpu'] = axes[i, 0].plot([], [])[0]
            self.lines[f'{name}_mem'] = axes[i, 1].plot([], [])[0]
    
    def update(self, node_metrics):
        for name, metrics in node_metrics.items():
            x = list(range(len(metrics['gpu_util'])))
            
            self.lines[f'{name}_gpu'].set_data(x, metrics['gpu_util'])
            self.lines[f'{name}_mem'].set_data(x, metrics['mem_used'])
            
            self.fig.canvas.draw()