class LSTM(nn.Module):
    def __init__(self, vocab_size, embed_dim, hidden_dim, num_intent_labels, max_intents):
        super(LSTM, self).__init__()
        # 进行词嵌入
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        # 使用nn.LSTM来进行序列建模，其中，input_size为词向量的维度，hidden_size为LSTM的隐层维度，num_layers为LSTM的层数，batch_first为True表示输入和输出的格式为(batch_size, seq_len, input_size)，bidirectional为True表示使用双向LSTM
        self.lstm = nn.LSTM(input_size=embed_dim, hidden_size=hidden_dim, num_layers=1, batch_first=True, bidirectional=True)
        
        # 预测意图的数量，使用线性层，输入为双向LSTM的输出最后一层(双向所以要乘2)，输出为max_intents维度
        self.intent_num_classifier = nn.Linear(hidden_dim * 2, max_intents)
        
        # 预测意图标签的分布，使用线性层，输入为双向LSTM的输出最后一层，输出为num_intent_labels维度
        self.intent_classifier = nn.Linear(hidden_dim * 2, num_intent_labels)
        
    def forward(self, input_ids):
        # 词嵌入
        embeds = self.embedding(input_ids)  # (batch_size, seq_len, embed_dim)
        
        # LSTM，这里的out为(batch_size, seq_len, hidden_dim * 2)，h_n和c_n为(num_layers * num_directions, batch_size, hidden_dim)
        out, (h_n, c_n) = self.lstm(embeds)
        
        h_last = torch.cat((h_n[-2], h_n[-1]), dim=1)  
        
        # 输出意图数量
        intent_count_logits = self.intent_num_classifier(h_last)  # (batch_size, max_intents)
        
        # 输出意图标签分布
        intent_logits = self.intent_classifier(h_last)  # (batch_size, num_intent_labels)

        return intent_count_logits, intent_logits

import torch
import torch.nn as nn

# 设置 batch_first=True
rnn = nn.RNN(input_size=4, hidden_size=2, batch_first=True)

# 假设有一个批量大小为3，序列长度为5，每个时间步有4个特征
input_data = torch.randn(3, 5, 4)  # shape: (batch_size, sequence_length, input_size)

# 进行前向传播
output, hidden = rnn(input_data)

print(output.shape)  # 输出形状: (batch_size, sequence_length, hidden_size)

batch_size_list = [32, 64]
lr_list = [1e-5, 1e-4, 1e-3]
embed_dim_list = [64, 128, 256]
hidden_dim_list = [64, 128]

# trainer.py

import torch
import numpy as np
from tqdm import tqdm
from seqeval.metrics import classification_report, precision_score, recall_score, f1_score
from sklearn.metrics import accuracy_score, precision_recall_fscore_support

def train_epoch(model, loader, optimizer, intent_count_loss_fn, intent_loss_fn, device): 
    # 训练模式
    model.train()
    total_loss = 0
    for batch in tqdm(loader, desc="Training", leave=False):
        optimizer.zero_grad()
        
        # 导入数据
        input_ids = batch['input_ids'].to(device)  # [batch_size, seq_len]
        intent_labels = batch['intent_labels'].to(device)  # [batch_size]
        intent_counts = batch['intent_counts'].to(device)  # [batch_size]
        
        ic_logits, it_logits = model(input_ids)  # [batch_size, num_intents]
        
        # 计算损失
        loss_ic = intent_count_loss_fn(ic_logits, intent_counts - 1)  # [batch_size]，注意这里counts要-1，因为分类任务通常是从0开始的
        loss_it = intent_loss_fn(it_logits, intent_labels)  # [batch_size]
        
        # 联合两个任务的loss进行联合训练
        loss = loss_ic + loss_it
        loss.backward()
        optimizer.step()
        
        total_loss += loss.item()
    avg_loss = total_loss / len(loader)
    print(f"训练损失: {avg_loss:.4f}")
    return avg_loss


def evaluate(model, loader, device, intent_count_loss_fn, intent_loss_fn, model_type="bert"):
    # eval不开梯度回传，关闭dropout等
    model.eval()
    
    intent_count_preds, intent_count_true = [], []
    intent_preds, intent_true = [], []
    all_true_bio, all_pred_bio = [], []
    
    with torch.no_grad():
        for batch in loader:
            input_ids = batch['input_ids'].to(device) # [batch_size, seq_len]
            intent_counts = batch['intent_counts'].to(device) # [batch_size] 
            intent_labels = batch['intent_labels'].to(device) # [batch_size， num_intent_labels], 0-1 编码, 1 表示存在该意图
            
            # ic_logits：[batch_size, max_intents], it_logits: [batch_size, num_intent_labels]
            ic_logits, it_logits = model(input_ids)
            
            ic_preds_batch = torch.argmax(ic_logits, dim=1).cpu().numpy()  # [batch_size]，只返回最大值的索引
            ic_true_batch = (intent_counts.cpu().numpy() - 1)  # [batch_size]，这个地方需要 -1，因为ic_preds_batch中的值从0开始的
            
            intent_count_preds.extend(ic_preds_batch)  # 将预测的意图数量添加到 intent_count_preds 列表中
            intent_count_true.extend(ic_true_batch)  # 将真实的意图数量添加到 intent_count_true 列表中

            it_logits_cpu = it_logits.cpu().numpy()  # [batch_size, num_intents]
            it_labels_cpu = intent_labels.cpu().numpy()  # [batch_size, num_intents]
            
            # 遍历这批数据的每个样本
            for i, logits in enumerate(it_logits_cpu):
                # 意图数量预测 的结果，表示模型预测该输入样本中包含的意图数量。由于 ic_preds_batch 中的值是从 0 开始的，因此加 1 是为了得到实际的意图数量
                N = ic_preds_batch[i] + 1
                # 如果预测的意图数量 N 大于 0，模型会继续处理并选择 N 个最有可能的意图。
                if N>0:
                    idxs = logits.argsort()[-N:][::-1] # 返回 it_logits 中每个样本对应的意图的排序索引（从最小到最大）
                    pred_vec = np.zeros_like(logits)  # 初始化一个与 logits 形状相同的零向量 pred_vec，用于保存每个样本的最终预测结果。
                    pred_vec[idxs] = 1  #对于预测的 N 个最可能的意图（由 idxs 确定），将对应的 pred_vec 元素设置为 1，表示这些是被模型识别为存在的意图。
                else:
                    pred_vec = np.zeros_like(logits)  # 如果 N 为 0，表示模型预测该样本没有任何意图，那么 pred_vec 将保持为全零。
                
                intent_preds.append(pred_vec)  #将生成的预测向量 pred_vec 添加到 intent_preds 列表中，表示该样本的意图预测
                intent_true.append(it_labels_cpu[i])  # 将真实标签（it_labels_cpu[i]）添加到 intent_true 列表中，表示该样本的真实意图标签。

    # 计算意图数量的准确率、精确度、召回率、F1值
    ic_acc = accuracy_score(intent_count_true, intent_count_preds)
    ic_pre, ic_rec, ic_f1, _ = precision_recall_fscore_support(
        intent_count_true, intent_count_preds, average='macro', zero_division=0
    )
    
    # 重新将列表转为numpy数组
    it_true_np = np.array(intent_true)
    it_pred_np = np.array(intent_preds)
    it_acc = accuracy_score(it_true_np, it_pred_np)
    # 调用sklearn的函数计算宏平均精度、精确度、召回率、F1值
    it_pre, it_rec, it_f1, _ = precision_recall_fscore_support(
        it_true_np, it_pred_np, average='macro', zero_division=0
    )
    
    print("\n=== 意图数量 ===")
    print(f"Acc: {ic_acc:.4f}, MacroP: {ic_pre:.4f}, MacroR: {ic_rec:.4f}, MacroF1: {ic_f1:.4f}")
    
    print("\n=== 意图识别 ===")
    print(f"Acc: {it_acc:.4f}, MacroP: {it_pre:.4f}, MacroR: {it_rec:.4f}, MacroF1: {it_f1:.4f}")
    
    
    return {
        'intent_count_accuracy': ic_acc,
        'intent_count_precision': ic_pre,
        'intent_count_recall': ic_rec,
        'intent_count_f1': ic_f1,
        'intent_accuracy': it_acc,
        'intent_precision': it_pre,
        'intent_recall': it_rec,
        'intent_f1': it_f1,
    }

intent_count_loss_fn = nn.CrossEntropyLoss()
intent_loss_fn = nn.BCEWithLogitsLoss()

loss_ic = intent_count_loss_fn(ic_logits, intent_counts - 1)
loss_it = intent_loss_fn(it_logits, intent_labels)

# 使用softmax + NLLloss实现CrossEntropyLoss
class CrossEntropyLoss(nn.Module):
    def __init__(self):
        super(CrossEntropyLoss, self).__init__()
        self.logsoftmax = nn.LogSoftmax(dim=1)  # 先进行softmax，再取Log
        self.nllloss = nn.NLLLoss()  # 负对数似然
    
    def forward(self, input, target):
        log_probs = self.logsoftmax(input)
        loss = self.nllloss(log_probs, target)
        return loss

# 使用sigmoid + BCEloss实现BCEWithLogitsLoss
class BCEWithLogitsLoss(nn.Module):
    def __init__(self):
        super(BCEWithLogitsLoss, self).__init__()
        self.sigmoid = nn.Sigmoid()  # 先进行sigmoid
        self.bceloss = nn.BCELoss()  # 再来BCELoss
    
    def forward(self, input, target):
        probs = self.sigmoid(input)
        loss = self.bceloss(probs, target)
        return loss

# main.py

import torch
import argparse
import yaml
from torch.utils.data import DataLoader

from trainer import train_epoch, evaluate
from models import (
    CNN,
    CNNSeq
)
from data_utils import SLUDataset  # 导入相关模块
import torch.nn as nn

intents_num = {'企业性质查询': 0,
          '营业利润查询': 1,
          '企业负债查询': 2,
          '项目成交状况查询': 3,
          '企业营业成本查询': 4,
          '小区绿化率查询': 5,
          '建筑密度查询': 6,
          '小区成交均价查询': 7,
          '营业总收入查询': 8,
          '地块总价查询': 9,
          '地块成交时间查询': 10,
          '企业债务违约查询': 11,
          '容积率查询': 12,
          '企业风险查询': 13,
          '地块归属查询': 14,
          '项目开发商信息查询': 15
          }

def main():
    # 解析命令行参数
    parser = argparse.ArgumentParser(description="SLU main entry")
    parser.add_argument('--config', type=str, default='configs/config.yaml', help='Path to config file')
    parser.add_argument('-m', '--model_type', type=str, default='cnn', choices=['cnn', 'cnn_seq'])
    args = parser.parse_args()
    
    # 读取args参数中的config路径对应的yaml文件
    with open(args.config, 'r', encoding='utf-8') as f:
        cfg = yaml.safe_load(f)
    
    # 读取yaml文件中的参数
    data_path = cfg['data_path']
    model_name = cfg.get('model_name', 'bert-base-chinese') # 取model_name参数，如果没有则取bert-base-chinese
    batch_size = cfg.get('batch_size', 16)  # 取batch_size参数，如果没有则取16
    epochs = cfg.get('epochs', 3)  # 取epochs参数，如果没有则取3
    lr = cfg.get('lr', 2e-5)  # 取lr参数，如果没有则取2e-5
    # 取其他参数，如果没有则取默认值
    vocab_size = cfg.get('vocab_size', 30522)  # 词表大小
    embed_dim = cfg.get('embed_dim', 128)  # 词向量维度

    print("加载预处理后的数据:", data_path)
    # 加载预处理后的数据
    data = torch.load(data_path)
    train_encodings = data['train']
    test_encodings = data['test']
    
    # 创建Dataset和DataLoader
    train_dataset = SLUDataset(train_encodings)
    test_dataset = SLUDataset(test_encodings)
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
    
    print(f"训练集: {len(train_dataset)}, 测试集: {len(test_dataset)}")
    
    # 设置模型类型
    print("选择模型类型:", args.model_type)
    
    # 设置在cpu还是gpu上运行
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    
    # 选择模型类型
    if args.model_type == 'cnn':
        model = CNN(
            vocab_size=vocab_size,
            embed_dim=embed_dim,
            num_intent_labels=len(intents_num),
        )
    elif args.model_type == 'cnn_seq':
        model = CNNSeq(
            vocab_size=vocab_size,
            embed_dim=embed_dim,
            num_intent_labels=len(intents_num),
    )
    
    # 加载模型到device
    model.to(device)
    
    # 定义损失函数
    intent_loss_fn = nn.CrossEntropyLoss()
    # 优化器选择AdamW
    optimizer = torch.optim.AdamW(model.parameters(), lr=float(lr))
    
    # 训练模型
    for epoch in range(epochs):
        print(f"\nEpoch {epoch+1}/{epochs} - {args.model_type}")
        train_epoch(model, train_loader, optimizer, intent_loss_fn, device)
        evaluate(model, test_loader, device, intent_loss_fn, args.model_type)
    
    # 保存模型
    ckpt_path = f"checkpoints/{args.model_type}_model.pth"
    torch.save(model.state_dict(), ckpt_path)
    print(f"模型已保存到 {ckpt_path}")

if __name__ == "__main__":
    main()

# data_preprocessing.py

import json
import torch
from transformers import BertTokenizerFast
import os

intents_num = {'企业性质查询': 0,
          '营业利润查询': 1,
          '企业负债查询': 2,
          '项目成交状况查询': 3,
          '企业营业成本查询': 4,
          '小区绿化率查询': 5,
          '建筑密度查询': 6,
          '小区成交均价查询': 7,
          '营业总收入查询': 8,
          '地块总价查询': 9,
          '地块成交时间查询': 10,
          '企业债务违约查询': 11,
          '容积率查询': 12,
          '企业风险查询': 13,
          '地块归属查询': 14,
          '项目开发商信息查询': 15
          }

def load_json(file_path):
    with open(file_path, 'r', encoding='utf-8') as f:
        data = json.load(f)  # 加载json文件
    
    processed_data = []
    for idx, example in enumerate(data):
        intent_str = example.get('intent', 'None')  # 取出意图
        tokens = example.get('tokens', [])  # 取出tokens
        
        # 处理意图，多个意图用“+”隔开，取第一个
        intents = [intent.strip() for intent in intent_str.split('+')]
        intent = intents[0]
        
        # 处理processed_data
        processed_data.append({
            'tokens': tokens,
            'intent': intent,
        })
    return processed_data

def encode_data(data, tokenizer, max_len=128):
    input_ids = []  
    intent_labels = []  

    # 遍历json文件数据
    for item in data:
        tokens = item['tokens']
        intent = item['intent']
        
        # 编码tokens，使用BertTokenizerFast进行tokenize
        encoding = tokenizer(
            tokens,
            is_split_into_words=True,  # 输入已经分词
            padding='max_length',  # 填充到最大长度
            truncation=True,
            max_length=max_len,
            return_offsets_mapping=True,
            return_tensors='pt'
        )
        # 将token编码后的input_ids添加到input_ids
        input_ids.append(encoding['input_ids'][0]) 

        # 单标签意图编码
        intent_id = intents_num.get(intent, -1)  # 找到对应的id
        intent_labels.append(torch.tensor(intent_id, dtype=torch.long))  # 添加到列表
        
    return {
        'input_ids': torch.stack(input_ids),
        'intent_labels': torch.stack(intent_labels),
    }

def main():
    train_file = 'train.json'
    val_file = 'val.json'
    test_file = 'test.json'
    save_path = 'data_preprocessed.pth'
    
    if not os.path.exists(train_file) or not os.path.exists(test_file):
        print("Error: train.json or test.json not found.")
        return
    
    # 分别加载数据
    train_data = load_json(train_file)
    val_data = load_json(val_file)
    test_data = load_json(test_file)
    
    # 加载tokenizer
    tokenizer = BertTokenizerFast.from_pretrained('../../bert-base-chinese') ## 如果是正常下载的huggingface版本，则直接引用“bert-base-chinese”，如果下载到本地，则引用本地地址
    
    # 分别编码数据
    print("Encoding training data...")
    train_encodings = encode_data(train_data, tokenizer)
    print("Encoding validation data...")
    val_encodings = encode_data(val_data, tokenizer)
    print("Encoding test data...")
    test_encodings = encode_data(test_data, tokenizer)

    # 保存数据，以pth格式保存
    torch.save({
        'train': train_encodings,
        'val': val_encodings,
        'test': test_encodings,
    }, save_path)
    print(f"Preprocessed data saved to {save_path}")
    
if __name__ == "__main__":
    main()

input_ids = [
    torch.tensor([101, 2023, 2003, 1037, 2742]),  # 第一个句子
    torch.tensor([101, 2023, 2003, 1037, 2742, 102]),  # 第二个句子
    torch.tensor([101, 2023, 2003, 2742, 102])  # 第三个句子
]

stacked_tensor = torch.stack(input_ids)
print(stacked_tensor)

import torch
from torch.utils.data import Dataset

class SLUDataset(Dataset):
    # 继承自Dataset类，用于加载数据集，对__len__和__getitem__方法进行重写
    def __init__(self, encodings):
        self.encodings = encodings
    
    def __len__(self):
        return len(self.encodings['input_ids'])  # 返回 input_ids键对应的值(一个一维tensor)的长度
    
    def __getitem__(self, idx):
        # item = {key: self.encodings[key][idx] for key in 
        #         ['input_ids']}
        # item['intent_labels'] = self.encodings['intent_labels'][idx]
        # return item
        # 这样好理解一点
        return {
            'input_ids': self.encodings['input_ids'][idx],
            'intent_labels': self.encodings['intent_labels'][idx]
        }

encodings = {
    'input_ids': torch.tensor([[101, 2345, ...], ...]),  # 假设有100个样本
    'intent_labels': torch.tensor([0, 3, 8, ...])       # 100个标签
}
dataset = SLUDataset(encodings)

def __getitem__(self, idx):
    # 构建样本字典
    item = {
        'input_ids': self.encodings['input_ids'][idx],  # 取第idx个样本的token序列
        'intent_labels': self.encodings['intent_labels'][idx]  # 对应标签
    }
    return item

# 假设idx=5
item = {
    'input_ids': tensor([ 101,  2345,  3456, ...,     0]),  # 长度=max_seq_len
    'intent_labels': tensor(3)  # 数值型标签
}

from torch.utils.data import DataLoader

# 创建数据集实例
dataset = SLUDataset(encodings)

# 创建DataLoader
loader = DataLoader(
    dataset,
    batch_size=16,
    shuffle=True
)

# 迭代获取批次数据
for batch in loader:
    print(batch['input_ids'].shape)    # torch.Size([16, 128])
    print(batch['intent_labels'].shape # torch.Size([16])

# 查看单个样本
sample = dataset[10]
print(f"Token IDs: {sample['input_ids']}")
print(f"Intent Label: {sample['intent_labels']}")

# 统计数据集信息
print(f"总样本数: {len(dataset)}")
print(f"输入维度: {dataset[0]['input_ids'].shape}")
print(f"标签示例: {dataset[0]['intent_labels'].item()}")

# models.py

import torch
from torch import nn

#######################################
# CNN原始模型
#######################################

class CNN(nn.Module):
    def __init__(self, vocab_size, embed_dim, num_intent_labels):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim) # 词嵌入层，输入为词索引，输出为词向量
        self.pool = nn.AdaptiveMaxPool1d(1) # 池化层，输入为词向量，输出为句子的最大池化结果
        
        self.conv1 = nn.Sequential(
            nn.Conv1d(in_channels=embed_dim, out_channels=embed_dim, kernel_size=3, padding=1),
            nn.BatchNorm1d(embed_dim),
            nn.ReLU(),
            nn.Dropout(0.5)
        )
        
        self.conv2 = nn.Sequential(
            nn.Conv1d(in_channels=embed_dim, out_channels=embed_dim * 2, kernel_size=3, padding=1),
            nn.BatchNorm1d(embed_dim * 2),
            nn.ReLU(),
        )
        
        self.conv3 = nn.Sequential(
            nn.Conv1d(in_channels=embed_dim * 2, out_channels=embed_dim, kernel_size=3, padding=1),
            nn.BatchNorm1d(embed_dim),
            nn.ReLU(),
        )
        
        self.intent_classifier = nn.Linear(embed_dim, num_intent_labels) # 分类层，输入为句子的最大池化结果，输出为意图标签的概率分布

    def forward(self, input_ids):
        '''
        input_ids: [batch_size, seq_len]，这是在传入时就设置的，即模型的输入
        '''
        
        # 词嵌入层
        embeds = self.embedding(input_ids) # [batch_size, seq_len, embed_dim]
        embeds = embeds.permute(0, 2, 1) # [batch_size, embed_dim, seq_len]  转换为CNN输入的格式，需要卷积的维度放在后面，为seq_len
        
        # 卷积层
        conv1_out = self.conv1(embeds) # [batch_size, embed_dim, seq_len]
        conv2_out = self.conv2(conv1_out) # [batch_size, embed_dim * 2, seq_len]
        conv3_out = self.conv3(conv2_out) # [batch_size, embed_dim, seq_len]
        
        # 池化层
        pooled = self.pool(conv3_out) # [batch_size, embed_dim, 1]
        result = pooled.squeeze(-1)  # [batch_size, embed_dim] 去除最后一个维度
        
        # 分类
        it_logits = self.intent_classifier(result)  # [batch_size, num_intent_labels]
        
        return it_logits

#######################################
# CNNSeq原始模型
#######################################

class CNNSeq(nn.Module):
    def __init__(self, vocab_size, embed_dim, num_intent_labels):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim) # 词嵌入层，输入为词索引，输出为词向量
        
        self.cnn_2 = nn.Conv1d(in_channels=embed_dim, out_channels=embed_dim // 4, kernel_size=2)  
        self.cnn_3 = nn.Conv1d(in_channels=embed_dim, out_channels=embed_dim // 4, kernel_size=3)
        self.cnn_4 = nn.Conv1d(in_channels=embed_dim, out_channels=embed_dim // 4, kernel_size=4)
        self.cnn_5 = nn.Conv1d(in_channels=embed_dim, out_channels=embed_dim // 4, kernel_size=5)
        
        self.pool = nn.AdaptiveMaxPool1d(1) # 池化层，输入为词向量，输出为句子的最大池化结果
        
        self.intent_classifier = nn.Linear(embed_dim, num_intent_labels) # 分类层，输入为句子的最大池化结果，输出为意图标签的概率分布

    def forward(self, input_ids):
        # 词嵌入
        embeds = self.embedding(input_ids) 
        embeds = embeds.permute(0, 2, 1) # [batch_size, embed_dim, seq_len]  转换为CNN输入的格式，需要卷积的维度放在后面，为seq_len
        
        # 卷积
        cout_2 = self.cnn_2(embeds)   # [batch_size, embed_dim // 4, seq_len - 1]
        pool_2 = self.pool(cout_2)    # [batch_size, embed_dim // 4, 1]
        
        cout_3 = self.cnn_3(embeds)   # [batch_size, embed_dim // 4, seq_len - 2]
        pool_3 = self.pool(cout_3)    # [batch_size, embed_dim // 4, 1]
        
        cout_4 = self.cnn_4(embeds)   # [batch_size, embed_dim // 4, seq_len - 3]
        pool_4 = self.pool(cout_4)    # [batch_size, embed_dim // 4, 1]
        
        cout_5 = self.cnn_5(embeds)   # [batch_size, embed_dim // 4, seq_len - 4]
        pool_5 = self.pool(cout_5)    # [batch_size, embed_dim // 4, 1]
        
        # 拼接
        pooled = torch.cat([pool_2.squeeze(-1), pool_3.squeeze(-1), pool_4.squeeze(-1), pool_5.squeeze(-1)], dim=1) # [batch_size, embed_dim]
        
        # 分类
        it_logits = self.intent_classifier(pooled)  # [batch_size, num_intent_labels]
        
        return it_logits

# 输入文本（已分词）
tokens = ['请', '问', '在', '南',...]  # 假设长度=47

# 经过tokenizer编码（max_len=128）
input_ids = [101, 2345, 567, 890,..., 0, 0]  # 长度128的向量

# 进入DataLoader后
batch = {
    'input_ids': torch.Size([16, 128]),  # 假设batch_size=16
    'intent_labels': torch.Size([16])
}

# 经过Embedding层（embed_dim=128）
embeds = embedding(input_ids)  # [16, 128, 128]

# CNN模型中的维度转换：
embeds.permute(0, 2, 1)  # [16, 128, 128] → [16, 128, 128]（此处可能设计不合理）
conv1_out = self.conv1(embeds)  # 保持[16, 128, 128]
pooled = self.pool(conv3_out)   # [16, 128, 1]
result = pooled.squeeze(-1)     # [16, 128]

# trainer.py

import torch
import numpy as np
from tqdm import tqdm
from seqeval.metrics import classification_report, precision_score, recall_score, f1_score
from sklearn.metrics import accuracy_score, precision_recall_fscore_support

def train_epoch(model, loader, optimizer, intent_loss_fn, device):
    # 训练模式
    model.train()
    total_loss = 0
    for batch in tqdm(loader, desc="Training", leave=False):
        optimizer.zero_grad()
        # 导入数据
        input_ids = batch['input_ids'].to(device)  # [batch_size, seq_len]
        intent_labels = batch['intent_labels'].to(device)  # [batch_size]
        
        it_logits  = model(input_ids)  # [batch_size, num_intents]
        
        # 计算损失
        loss_it = intent_loss_fn(it_logits, intent_labels)
        loss_it.backward()
        optimizer.step()
        
        total_loss += loss_it.item()
        
    avg_loss = total_loss / len(loader)
    print(f"训练损失: {avg_loss:.4f}")
    return avg_loss


def evaluate(model, loader, device, intent_loss_fn, model_type="cnn"):
    # eval不开梯度回传，关闭dropout等
    model.eval()
    
    intent_preds, intent_true = [], []
    
    with torch.no_grad():
        for batch in loader:
            input_ids = batch['input_ids'].to(device)  # [batch_size, seq_len]
            intent_labels = batch['intent_labels'].to(device)  # [batch_size]
            
            it_logits  = model(input_ids)  # [batch_size, num_intents]
            
            # print('label size: ', intent_labels.shape)
            # print('pred size: ', it_logits.shape)
            it_logits_cpu = it_logits.cpu().numpy().argmax(axis=1)  # [batch_size]，取最大值索引作为预测标签
            it_labels_cpu = intent_labels.cpu().numpy()  # [batch_size]， 真实标签
            
            intent_preds.extend(it_logits_cpu)  # 使用extend方法将预测标签添加到列表中
            intent_true.extend(it_labels_cpu)
            # print('label cpu size: ', intent_labels.shape)
            # print('pred cpu size: ', it_logits.shape)
    
    # 重新将列表转为numpy数组
    it_true_np = np.array(intent_true)
    it_pred_np = np.array(intent_preds)
    # print(it_true_np)
    # print(it_pred_np)
    it_acc = accuracy_score(it_true_np, it_pred_np)
    # 调用sklearn的函数计算宏平均精度、精确度、召回率、F1值
    it_pre, it_rec, it_f1, _ = precision_recall_fscore_support(
        it_true_np, it_pred_np, average='macro', zero_division=0
    )
    
    print("\n=== 意图识别 ===")
    print(f"Acc: {it_acc:.4f}, MacroP: {it_pre:.4f}, MacroR: {it_rec:.4f}, MacroF1: {it_f1:.4f}")
    
    return {
        'intent_accuracy': it_acc,
        'intent_precision': it_pre,
        'intent_recall': it_rec,
        'intent_f1': it_f1,
    }

model.train()  # 切换到训练模式
for data, target in train_loader:
    optimizer.zero_grad()
    output = model(data)
    loss = criterion(output, target)
    loss.backward()
    optimizer.step()  # 参数更新

model.eval()  # 切换到评估模式
with torch.no_grad():  # 关闭梯度计算
    for data, target in val_loader:
        output = model(data)
        accuracy = calculate_accuracy(output, target)

from gensim.models import Word2Vec
from gensim.utils import simple_preprocess

# 读入文件与预处理
with open('homework3_input.txt', encoding = 'utf-8') as f:
    text = f.read()

sentences = text.split('\n')

corpus = [sentence.split(' ') for sentence in sentences]
# print(corpus)
# print(type(corpus))

# 传入的是二维列表！
# corpus = [simple_preprocess(sentence) for sentence in sentences]
# print(corpus)

# 模型训练
model = Word2Vec(sentences=corpus, vector_size=100, window=5, min_count=1, workers=4, epochs=5, min_alpha=0.005, sg=0)

# 模型保存
model.save("word2vec.model")

# 模型加载
model = Word2Vec.load("word2vec.model")

# 相似度计算函数
def similarity_compute(word1, word2):
    similarity = model.wv.similarity(word1, word2)
    print(f"{word1} 和 {word2} 相似度为：{similarity}")

# 找到前5个相近词
def most_similar(word):
    simi_words = model.wv.most_similar(word, topn=5)
    print(f"{word}的前五个相近词：")
    for ans, score in simi_words:
        print(f"{ans} : {score}")

# 计算5组词之间的相似度
similarity_compute('俄罗斯', '乌克兰')
similarity_compute('舅舅', '叔叔')
similarity_compute('证券', '金融')
similarity_compute('足球', '篮球')
similarity_compute('火锅', '麻辣烫')

# 寻找相近词
most_similar('足球')
most_similar('新华社')
most_similar('舅舅')
most_similar('欧洲')
most_similar('心脏')

# 计算'爸爸-男人+女人'的结果
result = model.wv.most_similar(positive=['爸爸', '女人'], negative=['男人'], topn=5)
for word, score in result:
    print(f"可能的答案: {word}, 相似度：{score}")

import torch
from transformers import BertTokenizer
from transformers import BertModel
from sklearn.metrics.pairwise import cosine_similarity

# 中文Bert
model_name = '../第二周/bert-base-chinese/bert-base-chinese'
tokenizers = BertTokenizer.from_pretrained(model_name)
model = BertModel.from_pretrained(model_name)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  # 这句话不要忘了是字符串...

model.to(device)  # 将模型移动到指定的设备（GPU 或 CPU）
model.eval()  # 将模型设置为评估模式（关闭 dropout 等训练时的特殊操作）

def get_sentence_embedding(sentence):
    inputs = tokenizers(sentence, return_tensors='pt', truncation=True, max_length=128) # 
    inputs = {k: v.to(device) for k,v in inputs.items()} # 将输入数据移动到指定的设备（GPU 或 CPU）

    with torch.no_grad():
        outputs = model(**inputs) # 函数解包，将inputs传给模型，得到输出

    cls_embedding = outputs.last_hidden_state[:, 0, :]  # 取出[CLS]对应的向量作为句子的embedding
    return cls_embedding.cpu().numpy()  # 将embedding从GPU转回CPU，并转为numpy数组返回

def compute_sentence_similarity(sentence1, sentence2):
    embedding1 = get_sentence_embedding(sentence1)  # 这里的embedding1就是句子1的embedding
    embedding2 = get_sentence_embedding(sentence2)  # 这里的embedding2就是句子2的embedding

    sim = cosine_similarity(embedding1, embedding2)[0][0]  # 会返回如$$[0.9926]] 这样的(1,1)矩阵结果，需要用[0][0]进行提取
    print(f"句子相似度: {sim:.4f}")

sentence1 = "中国队大胜尼日利亚队"
sentence2 = "尼日利亚队大败中国队"
sentence3 = '吃葡萄不吐葡萄皮'
sentence4 = '不吃葡萄倒吐葡萄皮'
sentence5 = '夏洛特烦恼'
sentence6 = '西红市首富赚了100万，于是开了开心麻花'
sentence7 = '他正在学习自然语言处理'
sentence8 = '他正在学习人工智能技术'
sentence9 = '这部电影非常精彩，值得一看'
sentence10 = '这部电影太无聊了，浪费时间'

compute_sentence_similarity(sentence1, sentence2)
compute_sentence_similarity(sentence3, sentence4)
compute_sentence_similarity(sentence5, sentence6)
compute_sentence_similarity(sentence7, sentence8)
compute_sentence_similarity(sentence9, sentence10)

# 英文Bert
model_name = '../第二周/bert-base-cased/bert-base-cased'
tokenizers = BertTokenizer.from_pretrained(model_name)
model = BertModel.from_pretrained(model_name)

'''中间省略'''

sentence1 = "The weather is nice today, and the sun is shining."
sentence2 = "I love you."
sentence3 = "I enjoy eating apples and bananas."
sentence4 = "Apples and bananas are my favorite fruits."
sentence5 = "He is studying natural language processing."
sentence6 = "The cat is sleeping."
sentence7 = "This movie is fantastic and worth watching."
sentence8 = "That bookis so boring and a waste of time."
sentence9 = "She went to Beijing yesterday."
sentence10 = "He has married a beautiful girl."

import torch
from transformers import GPT2Tokenizer
from transformers import GPT2Model
from sklearn.metrics.pairwise import cosine_similarity

model_name = './gpt2'
tokenizers = GPT2Tokenizer.from_pretrained(model_name)
model = GPT2Model.from_pretrained(model_name)

inputs = {
    'input_ids': tensor1,
    'attention_mask': tensor2
}

input_ids=tensor1, attention_mask=tensor2

outputs = model(**inputs)

outputs = model(input_ids=inputs['input_ids'], attention_mask=inputs['attention_mask'])

inputs = {
    'input_ids': torch.tensor([[1, 2, 3]]),
    'attention_mask': torch.tensor([[1, 1, 1]])
}

model(input_ids=torch.tensor([[1, 2, 3]]), attention_mask=torch.tensor([[1, 1, 1]]))

import torch
import torch.nn as nn

# 定义CBOW模型
class CBOWModel(nn.Module):
    def __init__(self, vocab_size, embedding_dim):
        '''参数初始化'''
        super(CBOWModel, self).__init__() # 继承父类(nn.Module)的初始化方法
        self.vocab_size = vocab_size  # 词汇表大小
        self.embedding_dim = embedding_dim  # 词向量维度
        self.input_embeddings = nn.Embedding(vocab_size, embedding_dim)  # 输入词向量嵌入层
        self.output_embeddings = nn.Embedding(vocab_size, embedding_dim)  # 输出词向量嵌入层
        self.init_emb()  # 初始化词向量

    def init_emb(self):
        initrange = 0.5 / self.embedding_dim  # 随机初始化范围
        self.input_embeddings.weight.data.uniform_(-initrange, initrange)  # 输入词向量初始化, uniform_()方法用于将权重张量初始化为均匀分布
        self.output_embeddings.weight.data.uniform_(-0, 0)  # 输出词向量初始化

    def forward(self, context_words, target_words, negative_words):
        '''
        context_words: 上下文词的索引，形状为 (batch_size, context_size)
        target_words: 目标词的索引，形状为 (batch_size)
        negative_words: 负采样的词索引，形状为 (batch_size, num_negatives)
        '''
        # 将上下文词通过输入嵌入层转换为向量，得到形状 (batch_size, context_size, embedding_dim)
        context_emb = self.input_embeddings(context_words)  # (batch, context_size, embed_dim) 
        # 对上下文词的嵌入取平均，得到单个向量表示整个上下文，形状变为 (batch_size, embedding_dim)
        context_emb = torch.mean(context_emb, dim=1)  # (batch, embed_dim)
        # 目标词和负样本通过输出嵌入层转换为向量，形状分别为 (batch_size, embedding_dim) 和 (batch_size, num_negatives, embedding_dim)。
        target_emb = self.output_embeddings(target_words)  # (batch, embed_dim)
        negative_emb = self.output_embeddings(negative_words)  # (batch, neg, embed_dim)

        # 计算正样本的损失
        # 分数计算：上下文向量与目标词向量的逐元素乘积之和（点积），表示两者的相似性。
        positive_score = torch.sum(context_emb * target_emb, dim=1)  # (batch)
        # 损失转换：通过sigmoid将分数映射到(0,1)区间，取对数并添加极小值（1e-10）防止数值溢出。
        positive_loss = torch.log(torch.sigmoid(positive_score) + 1e-10)

        # 计算负样本的损失
        # 分数计算：context_emb.unsqueeze(2) 扩展维度为 (batch, embed_dim, 1)，torch.bmm（批量矩阵乘法）计算每个负样本与上下文向量的点积，得到形状 (batch_size, num_negatives)
        negative_score = torch.bmm(negative_emb, context_emb.unsqueeze(2)).squeeze()  # (batch, neg)
        # 损失转换：对负样本分数取负号（因为希望负样本得分低），通过sigmoid和log计算损失，并对所有负样本求和。
        negative_loss = torch.sum(torch.log(torch.sigmoid(-negative_score) + 1e-10), dim=1)  # (batch)

        loss = -(positive_loss + negative_loss) # (batch)，负号是为了最小化loss
        return loss.mean()  # 平均loss

negative_score = torch.bmm(negative_emb, context_emb.unsqueeze(2)).squeeze()  # (batch, neg)
negative_loss = torch.sum(torch.log(torch.sigmoid(-negative_score) + 1e-10), dim=1)

loss = -(positive_loss + negative_loss).mean()

import numpy as np

# 负采样分布
def get_negative_sampling_distribution(word_counts):
    # 使用词频的0.75次方
    power = 0.75
    print(word_counts)
    distribution = word_counts ** power  # 计算负采样分布，给词频乘以0.75次方，进行平滑处理
    print(distribution)
    distribution /= distribution.sum()  # 归一化
    print(distribution)
    return distribution  # 返回负采样分布

class NegativeSampler:
    def __init__(self, distribution):
        self.distribution = distribution  # 初始化分布
        self.vocab_size = len(distribution)  # 设置词汇表大小就是分布大小

    def sample(self, batch_size, num_negatives):
        # 使用numpy的choice进行批量采样
        return np.random.choice(self.vocab_size, size=(batch_size, num_negatives), p=self.distribution)  # 按照负采样分布采样，通过设置每个词的采样概率p实现，返回的是一个二维数组，shape=(batch_size, num_negatives)，采样的范围是 [0, vocab_size)，其中，二维数组的每一行表示每个正样本对应的负样本索引

import torch
import numpy as np
from torch import optim
from tqdm import tqdm

class Trainer:
    def __init__(self, model, config, dataloader, word2idx, idx2word, sampler):
        # 参数初始化
        self.model = model  
        self.config = config
        self.dataloader = dataloader
        self.word2idx = word2idx
        self.idx2word = idx2word
        self.sampler = sampler
        
        # 设备设置
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self.model.to(self.device)
        
        # 优化器设置
        self.optimizer = optim.Adam(self.model.parameters(), lr=config.learning_rate)
        
        self.loss_history = []  # 记录损失
        
    def _train_epoch(self, epoch):
        '''训练单轮epoch'''
        train_loss = 0   # 初始化训练损失
        progress_bar = tqdm(self.dataloader, desc=f"Epoch {epoch+1}/{self.config.epochs}")  # 进度条
        for batch in progress_bar:  
            if self.config.mode == 'skip-gram':  # skip-gram模式
                center, target = batch  # skip-gram模式,sentences为中心词和目标词对
                inputs = torch.tensor(center, dtype=torch.long).to(self.device)  # 转换为tensor
            else:
                context, target = batch  # cbow模式,sentences为上下文词构成的元组和目标词对
                inputs = torch.tensor(context, dtype=torch.long).to(self.device)  # 转换为tensor
            
            targets = torch.tensor(target, dtype=torch.long).to(self.device)  # 将目标词转换为tensor
            batch_size = targets.size(0)  # 批量大小，即中心词个数
            
            # 负采样
            negative_samples = self.sampler.sample(batch_size, self.config.num_negatives)  # 进行负采样，其中负采样的个数为num_negatives，结果是一个batch_size*num_negatives的矩阵
            negative_samples = torch.tensor(negative_samples, dtype=torch.long).to(self.device) # 将负采样结果转换为tensor
            
            # 清空原本的梯度，进行前向传播
            self.optimizer.zero_grad()
            loss = self.model(inputs, targets, negative_samples) # 计算损失
            
            # loss进行反向传播，更新优化器参数
            loss.backward()
            self.optimizer.step()
            
            # 记录损失
            train_loss += loss.item()
            # progress_bar.set_postfix(loss=loss.item())
        
        # 计算平均损失
        avg_loss = train_loss / len(self.dataloader)
        self.loss_history.append(avg_loss)  # 记录损失

        return avg_loss

    def train(self):
        for epoch in range(self.config.epochs):
            avg_loss = self._train_epoch(epoch)  # 训练单轮epoch
            print(f"Epoch {epoch+1}/{self.config.epochs}, Loss: {avg_loss:.4f}")

    def get_and_save(self):
        # 获取词向量
        embeddings = self.model.input_embeddings.weight.data.cpu().numpy() 
        
        # 保存词向量
        with open(self.config.save_path, 'w', encoding='utf-8') as f:
            for idx, word in self.idx2word.items():
                # map 是 Python 的内置函数，它会对一个可迭代对象（如列表）中的每个元素应用指定的函数（这里是 str 函数），并返回一个新的迭代器
                # join 是 Python 的内置函数，它会将序列中的元素以指定的字符连接起来，并返回一个新的字符串
                vector = ' '.join(map(str, embeddings[idx])) 
                f.write(f"{word} {vector}\n")  # 保存词以及对应的词向量
        print(f"Word vectors saved to {self.config.save_path}")

        # 构建词向量矩阵并进行归一化
        norm_embeddings = embeddings / (np.linalg.norm(embeddings, axis=1, keepdims=True) + 1e-10)

        return norm_embeddings
    

import torch
import random
import numpy as np
from torch.utils.data import DataLoader
from torch import optim
from utils.configs import Config
from utils.dataloader import Word2VecDataset, read_data, build_vocab
from utils.negative_sampler import NegativeSampler, get_negative_sampling_distribution
from utils.metrics import cosine_similarity, find_most_similar
from models.skip_gram import SkipGramModel
from models.cbow import CBOWModel
from trainers import Trainer

# 设置随机种子以确保结果可复现
torch.manual_seed(0)
random.seed(0)
np.random.seed(0)

def main():
    # 解析命令行参数
    config = Config()
    print("配置参数：")
    print(f"embedding_dim: {config.embedding_dim}")
    print(f"window_size: {config.window_size}")
    print(f"num_negatives: {config.num_negatives}")
    print(f"batch_size: {config.batch_size}")
    print(f"epochs: {config.epochs}")
    print(f"learning_rate: {config.learning_rate}")
    print(f"mode: {config.mode}")
    print(f"data_path: {config.data_path}")
    print(f"save_path: {config.save_path}")
    
    # 读取数据
    corpus = read_data(config.data_path)
    word2idx, idx2word, word_counts = build_vocab(corpus, min_count=1)  # 构建词汇表
    vocab_size = len(word2idx)  # 词汇表大小
    print(f"Vocab size: {vocab_size}")  # 打印词汇表大小
    
    # 负采样分布
    distribution = get_negative_sampling_distribution(word_counts)  # 根据词频构建分布
    sampler = NegativeSampler(distribution)  # 根据分布构建负采样器

    # 创建Dataset和DataLoader
    dataset = Word2VecDataset(corpus, word2idx, config.window_size, mode=config.mode)  # 创建Dataset
    dataloader = DataLoader(dataset, batch_size=config.batch_size, shuffle=True)  # 创建DataLoader

    # 初始化模型
    if config.mode == 'skip-gram':
        model = SkipGramModel(vocab_size, config.embedding_dim)  # 初始化Skip-gram模型
    else:
        model = CBOWModel(vocab_size, config.embedding_dim)  # 初始化CBOW模型

    # 初始化训练器
    trainer = Trainer(
        model=model,
        config=config,
        dataloader=dataloader,
        word2idx=word2idx,
        idx2word=idx2word,
        sampler=sampler
    )
    
    # 训练模型
    trainer.train()
    
    # 保存词向量，并获得归一化的词向量
    norm_embeddings = trainer.get_and_save()

    word_a = '新华社'  
    word_b = '法新社'  
    if word_a in word2idx and word_b in word2idx:  # 判断词是否在词汇表中
        vec_a = norm_embeddings[word2idx[word_a]]  # 先将词转成索引，再取出对应的词向量
        vec_b = norm_embeddings[word2idx[word_b]]
        similarity = cosine_similarity(vec_a, vec_b)  # 计算余弦相似度
        print(f"'{word_a}' 与 '{word_b}' 的相似度为: {similarity:.4f}")
    else:
        print(f"'{word_a}' 或 '{word_b}' 不在词汇表中。")

    # 示例：查找与指定词最相似的前N个词
    target_word = '搜狐'  
    top_n = 5
    similar_words = find_most_similar(target_word, word2idx, idx2word, norm_embeddings, top_n=top_n)  # 找到与target_word最相似的前N个词
    if similar_words:
        print(f"与 '{target_word}' 最相似的前 {top_n} 个词:")
        for word, score in similar_words:
            print(f"{word}: {score:.4f}")

if __name__ == "__main__":
    main() # 调用main函数