模型下载

# from modelscope import snapshot_download
# import os

# # 设置缓存目录到数据盘
# os.environ['MODELSCOPE_CACHE'] = '/root/autodl-tmp/modelscope_cache'

# # Llama-3.1-8B 在 ModelScope 的模型ID
# model_id = 'LLM-Research/Meta-Llama-3.1-8B'
# local_dir = '/root/autodl-tmp/llama-3.1-8b'

# print(f"开始从 ModelScope  下载 {model_id}...")

# try:
#     # 下载模型
#     model_dir = snapshot_download(
#         model_id=model_id,
#         cache_dir=local_dir,
#         revision='master'
#     )
#     print(f"✓ 模型已下载到: {model_dir}")
    
# except Exception as e:
#     print(f"下载失败: {e}")

模型使用

import torch
import os
import sys
from transformers.models.llama.modeling_llama import (
    LlamaAttention,
    LlamaDecoderLayer,
    LlamaForCausalLM,
)
from transformers import AutoTokenizer, AutoModelForCausalLM  # 修改这里
from smoothquant.smooth import smooth_lm
from smoothquant.fake_quant import W8A8Linear, quantize_llama_like
from datasets import load_dataset
from huggingface_hub import hf_hub_download
import json
import random
from datasets import Dataset
class Evaluator:
    def __init__(self, dataset, tokenizer, device):
        self.dataset = dataset
        self.tokenizer = tokenizer
        self.device = device

        # tokenize the dataset
        def tokenize_function(examples):
            example = self.tokenizer(examples["text"])
            return example

        self.dataset = self.dataset.map(tokenize_function, batched=True)
        self.dataset.set_format(type="torch", columns=["input_ids"])

    @torch.no_grad()
    def evaluate(self, model):
        model.eval()
        # The task is to predict the last word of the input.
        total, hit = 0, 0
        for batch in self.dataset:
            input_ids = batch["input_ids"].to(self.device).unsqueeze(0)
            label = input_ids[:, -1]
            outputs = model(input_ids)
            last_token_logits = outputs.logits[:, -2, :]
            pred = last_token_logits.argmax(dim=-1)
            total += label.size(0)
            hit += (pred == label).sum().item()
        acc = hit / total
        return acc

# 设置镜像
os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'

# 修改模型路径
model_path = "/root/autodl-tmp/llama-3.1-8b/LLM-Research/Meta-Llama-3.1-8B" 

# 修改 tokenizer 加载 - 使用 AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_path)
print("成功加载 LLaMA-3.1-8B tokenizer")

# 创建数据集函数（保持不变）
def create_lambada_style_dataset():
    """创建类似 LAMBADA 风格的合成数据集，特别适合最后一个词预测任务"""
    
    # 创建有明确上下文和可预测结尾的句子
    samples = []
    
    # 模板1：明显的前后文关系
    context_completion_pairs = [
        # 人物动作
        ("She opened the book and started reading the", "story"),
        ("He picked up the guitar and began to play a", "song"),
        ("The chef took out his knife and began to chop the", "vegetables"),
        ("The artist looked at the blank canvas and decided to paint a", "landscape"),
        
        # 自然现象
        ("The sun was setting behind the mountains and the sky turned", "orange"),
        ("As the storm approached, people began to board up their", "windows"),
        ("The river flowed gently through the valley towards the", "sea"),
        ("The leaves changed color in the autumn and fell from the", "trees"),
        
        # 日常活动
        ("After finishing his homework, he turned off the", "light"),
        ("She put on her coat and went outside to catch the", "bus"),
        ("He studied for many hours to prepare for the important", "exam"),
        ("They packed their bags and got ready for their", "trip"),
        
        # 职业相关
        ("The detective examined the evidence and concluded that the", "butler"),
        ("The scientist conducted experiments to test her", "hypothesis"),
        ("The teacher wrote the lesson plan on the", "blackboard"),
        ("The programmer debugged the code to fix the", "error"),
        
        # 情感与关系
        ("After many years apart, they finally reunited and hugged each", "other"),
        ("She was so happy that she couldn't stop", "smiling"),
        ("He felt nervous before giving his first public", "speech"),
        ("They worked together to complete the difficult", "project"),
        
        # 时间与顺序
        ("First, he boiled the water. Then he added the", "pasta"),
        ("In the morning, she always drinks a cup of", "coffee"),
        ("Before going to bed, he always brushes his", "teeth"),
        ("After the movie ended, they discussed the", "plot"),
        
        # 地点与方向
        ("They walked through the forest and reached a small", "cabin"),
        ("The car drove down the highway towards the", "city"),
        ("She climbed to the top of the mountain to see the", "view"),
        ("He sailed across the ocean to explore new", "lands"),
        
        # 原因与结果
        ("Because it was raining, she decided to take an", "umbrella"),
        ("Since he was hungry, he made himself a", "sandwich"),
        ("As the room was dark, she turned on the", "light"),
        ("Due to the traffic, they arrived", "late"),
    ]
    
    # 生成更多变体
    for context, completion in context_completion_pairs:
        # 创建多个变体
        for i in range(25):  # 每个模板生成25个变体
            # 轻微修改上下文
            modified_context = context
            if i > 0:
                # 添加一些变化
                variations = [
                    f"Yesterday, {context.lower()}",
                    f"Last week, {context.lower()}",
                    f"In the story, {context.lower()}",
                    f"Surprisingly, {context.lower()}",
                    f"Eventually, {context.lower()}",
                    f"Carefully, {context.lower()}",
                    f"Quickly, {context.lower()}",
                    f"Suddenly, {context.lower()}",
                    f"Finally, {context.lower()}",
                    f"Interestingly, {context.lower()}",
                ]
                if i <= len(variations):
                    modified_context = variations[i-1]
            
            # 创建完整的文本
            full_text = f"{modified_context} {completion}."
            samples.append({"text": full_text})
    
    # 确保有1000个样本
    while len(samples) < 1000:
        # 使用基础模板创建更多样本
        base_context, base_completion = random.choice(context_completion_pairs)
        variation = random.choice([
            f"Earlier that day, {base_context.lower()}",
            f"Without hesitation, {base_context.lower()}",
            f"After much thought, {base_context.lower()}",
            f"In the end, {base_context.lower()}",
            f"To everyone's surprise, {base_context.lower()}",
        ])
        full_text = f"{variation} {base_completion}."
        samples.append({"text": full_text})
    
    return Dataset.from_list(samples[:1000])

# 加载数据集
dataset = create_lambada_style_dataset()
print(f"数据集大小: {len(dataset)}")
print("\n前5个示例:")
for i in range(min(5, len(dataset))):
    print(f"{i+1}: {dataset[i]['text']}")

print("数据加载成功")
evaluator = Evaluator(dataset, tokenizer, "cuda")
print("评估器创建成功")

# # 修改 FP16 模型加载
# model_fp16 = LlamaForCausalLM.from_pretrained(
#     model_path, 
#     torch_dtype=torch.float16, 
#     device_map="auto"
# )
# print("LLaMA-3.1-8B 模型加载成功")

# # 评估原始模型
# acc_fp16 = evaluator.evaluate(model_fp16)
# print(f"Original LLaMA-2-7B model (fp16) accuracy: {acc_fp16}")

# # 量化函数调用
# model_w8a8 = quantize_llama_like(model_fp16)  
# print(model_w8a8)

# acc_w8a8 = evaluator.evaluate(model_w8a8)
# print(f"Naive W8A8 quantized LLaMA-2-7B model accuracy: {acc_w8a8}")

# SmoothQuant 部分
model = LlamaForCausalLM.from_pretrained(
    model_path, 
    torch_dtype=torch.float16, 
    device_map="auto"
)
print("LLaMA-3.1-8B 模型重新加载")

# 加载 LLaMA 的激活尺度文件
act_scales = torch.load("/root/act_scales/llama-2-7b.pt") 
print("LLaMA 平滑模型导入成功")

smooth_lm(model, act_scales, 0.5)
model_smoothquant_w8a8 = quantize_llama_like(model) 
print(model_smoothquant_w8a8)
    
acc_smoothquant_w8a8 = evaluator.evaluate(model_smoothquant_w8a8)
print(f"SmoothQuant W8A8 quantized LLaMA-3.1-8B model accuracy: {acc_smoothquant_w8a8}")