RNN.ipynb

{"cells":[{"cell_type":"code","execution_count":77,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":8056,"status":"ok","timestamp":1683736421825,"user":{"displayName":"张耒阳","userId":"04359299908287882141"},"user_tz":-480},"id":"CgRTaRd9-X1t","outputId":"53f3eb52-fe7a-4874-c3f8-2051eca6328b"},"outputs":[{"name":"stderr","output_type":"stream","text":["\n","\n","\n","100%|██████████| 10000/10000 [00:00<00:00, 142474.40it/s]\n"]}],"source":["# import json\n","# from torchtext.data import get_tokenizer\n","# import jieba\n","# from collections import Counter\n","\n","# def read_file(json_path):\n","#     english_sentences = []\n","#     chinese_sentences = []\n","#     tokenizer = get_tokenizer('basic_english')\n","#     with open(json_path, 'r') as fp:\n","#         for idx, line in enumerate(fp):\n","#             if idx >= 10000: # Only read first 500000 lines\n","#                  break\n","#             line = json.loads(line)\n","#             english, chinese = line['english'], line['chinese']\n","#             # print(english)\n","#             # print(chinese)\n","#             # Tokenize\n","#             english = tokenizer(english)\n","#             chinese = list(jieba.cut(chinese))\n","#             chinese = [x for x in chinese if x not in {' ', '\\t'}]\n","#             english_sentences.append(english)\n","#             chinese_sentences.append(chinese)\n","#     return english_sentences, chinese_sentences\n","\n","# fen, fzh = read_file('data/translation2019zh_train.json')\n","# zh_en = []\n","\n","# data_len = len(fen)\n","# for i in range(data_len):\n","#     zh_en.append([fzh[i], fen[i]])\n","# from tqdm import tqdm\n","\n","# zh_words = set()\n","# en_words = set()  # 初始化集合对象  自动去重\n","\n","# for s in tqdm(zh_en):\n","#     for w in s[0]: # 统计中文\n","#         if w:\n","#             zh_words.add(w)\n","#     for w in s[1]: # 统计英文\n","#         if w:\n","#             en_words.add(w)\n","# zh_wl = ['<sos>', '<eos>', '<pad>'] + list(zh_words)\n","# en_wl = ['<sos>', '<eos>', '<pad>'] + list(en_words)\n","# pad_id = 2\n","\n","# # step5 利用字典对象存储词和索引的对应关系\n","# en2id = {}\n","# zh2id = {}\n","# for i, w in enumerate(en_wl):  # 遍历枚举类型对象实现此功能\n","#     en2id[w] = i\n","# for i, w in enumerate(zh_wl):\n","#     zh2id[w] = i \n","# import random\n","\n","# random.shuffle(zh_en)  # 随机打乱全部数据\n","# train_num = int(len(zh_en) * 0.8)\n","# train_set = zh_en[:train_num]  # 8成用于训练\n","# dev_set = zh_en[train_num:]  # 2成用于测试\n","# import torch\n","\n","# batch_size = 16\n","# data_workers = 0  # 子进程数 \n","\n","# class MyDataSet(torch.utils.data.Dataset):\n","#     def __init__(self, examples):\n","#         self.examples = examples\n","\n","#     def __len__(self):\n","#         return len(self.examples)\n","\n","#     def __getitem__(self, index):\n","#         example = self.examples[index]\n","#         s1 = example[0]\n","#         s2 = example[1]\n","#         l1 = len(s1)\n","#         l2 = len(s2)\n","#         return s1, l1, s2, l2, index  # 英文句子  英文句子长度  中文句子  中文句子长度 当前数据在数据集中的索引\n","\n","# def the_collate_fn(batch):\n","#     src = [[0] * batch_size]  # src ---> source 缩写   该任务中 源句子指的是中文句子  # 每个样本的开头都是0(起始标识符的编码)\n","#     tar = [[0] * batch_size]  # tar ---> target 缩写           目标句子指的是英文句子\n","#     src_max_l = 100  # 初始化英文句子最大长度  方便计算需要填充的个数\n","#     src_max_length = 0\n","#     for b in batch: # 每个batch的数据有五个信息 分别是: 英文句子  英文句子长度  中文句子  中文句子长度 当前数据在数据集中的索引\n","#         if src_max_length < min(src_max_l, b[1]):  # b[1] 即英文句子的长度\n","#             src_max_length = min(src_max_l, b[1])\n","#     tar_max_l = 100\n","#     tar_max_length = 0\n","#     for b in batch:\n","#         if tar_max_length < min(tar_max_l,b[3]):  # b[3] 即中文句子的长度\n","#             tar_max_length = min(tar_max_l,b[3])\n","#     for i in range(src_max_length):\n","#         l = []\n","#         for x in batch:\n","#             if i < x[1]:\n","#                 l.append(zh2id[x[0][i]])\n","#             else:\n","#                 l.append(pad_id)  # 如果句子长度小于最大句子长度，进行填充\n","#         src.append(l)\n","#         # l记录的是每个句子的第 i 个词  有多少个句子？ batch size个，因此len(l) == batch_size == 句子的数量\n","#         # src记录的是每个 l  总共多少个l？ src_max_l个，因此len(src) == src_max_l == 句子的最大长度\n","#         # len(src) == 句子的最大长度    len(src[0]) == 句子的数量\n","#         # [len(src), len(src[0])] ==> [src len, batch size]\n","\n","#     for i in range(tar_max_length):  # 注释参考上面\n","#         l = []\n","#         for x in batch:\n","#             if i < x[3]:\n","#                 l.append(en2id[x[2][i]])\n","#             else:\n","#                 l.append(pad_id)  # 如果句子长度小于最大句子长度，进行填充\n","#         tar.append(l)\n","#     indexs = [b[4] for b in batch]  # b[4] 记录的是 当前数据在数据集中的索引\n","#     src.append([1] * batch_size)  # 终止标识符的编码为1 所以src和tar在句子的最后把终止符加上\n","#     tar.append([1] * batch_size)\n","#     s1 = torch.LongTensor(src) \n","#     s2 = torch.LongTensor(tar)\n","#     return s1, s2, indexs\n","# # 构建训练集 \n","# train_dataset = MyDataSet(train_set)\n","# dev_dataset = MyDataSet(dev_set)\n","# # 定义训练集数据加载器和验证集数据加载器\n","# train_data_loader = torch.utils.data.DataLoader(\n","#     train_dataset,\n","#     batch_size=batch_size,\n","#     shuffle=True,\n","#     num_workers=data_workers,\n","#     collate_fn=the_collate_fn,\n","# )\n","\n","# dev_data_loader = torch.utils.data.DataLoader(\n","#     dev_dataset,\n","#     batch_size=batch_size,\n","#     shuffle=True,\n","#     num_workers=data_workers,\n","#     collate_fn=the_collate_fn,\n","# )"]},{"cell_type":"code","execution_count":60,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Requirement already satisfied: torchtext==0.6.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (0.6.0)\n","Requirement already satisfied: tqdm in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torchtext==0.6.0) (4.64.1)\n","Requirement already satisfied: sentencepiece in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torchtext==0.6.0) (0.1.98)\n","Requirement already satisfied: torch in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torchtext==0.6.0) (2.0.0)\n","Requirement already satisfied: six in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torchtext==0.6.0) (1.16.0)\n","Requirement already satisfied: numpy in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torchtext==0.6.0) (1.24.2)\n","Requirement already satisfied: requests in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torchtext==0.6.0) (2.28.1)\n","Requirement already satisfied: charset-normalizer<3,>=2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests->torchtext==0.6.0) (2.1.1)\n","Requirement already satisfied: certifi>=2017.4.17 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests->torchtext==0.6.0) (2022.12.7)\n","Requirement already satisfied: idna<4,>=2.5 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests->torchtext==0.6.0) (3.4)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests->torchtext==0.6.0) (1.26.14)\n","Requirement already satisfied: typing-extensions in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torch->torchtext==0.6.0) (4.4.0)\n","Requirement already satisfied: sympy in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torch->torchtext==0.6.0) (1.11.1)\n","Requirement already satisfied: jinja2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torch->torchtext==0.6.0) (3.1.2)\n","Requirement already satisfied: filelock in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torch->torchtext==0.6.0) (3.9.0)\n","Requirement already satisfied: networkx in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from torch->torchtext==0.6.0) (2.8.4)\n","Requirement already satisfied: MarkupSafe>=2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from jinja2->torch->torchtext==0.6.0) (2.1.1)\n","Requirement already satisfied: mpmath>=0.19 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from sympy->torch->torchtext==0.6.0) (1.2.1)\n","Requirement already satisfied: einops in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (0.6.1)\n","Requirement already satisfied: spacy in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (3.5.2)\n","Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (3.0.8)\n","Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (1.0.4)\n","Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (4.64.1)\n","Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (2.0.8)\n","Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (2.4.6)\n","Requirement already satisfied: requests<3.0.0,>=2.13.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (2.28.1)\n","Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (1.0.9)\n","Requirement already satisfied: packaging>=20.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (23.0)\n","Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (8.1.10)\n","Requirement already satisfied: typer<0.8.0,>=0.3.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (0.7.0)\n","Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (6.3.0)\n","Requirement already satisfied: numpy>=1.15.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (1.24.2)\n","Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (3.3.0)\n","Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (2.0.7)\n","Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (1.1.1)\n","Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (1.10.7)\n","Requirement already satisfied: pathy>=0.10.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (0.10.1)\n","Requirement already satisfied: jinja2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (3.1.2)\n","Requirement already satisfied: setuptools in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (65.6.3)\n","Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy) (3.0.12)\n","Requirement already satisfied: typing-extensions>=4.2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy) (4.4.0)\n","Requirement already satisfied: certifi>=2017.4.17 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2022.12.7)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.26.14)\n","Requirement already satisfied: idna<4,>=2.5 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.4)\n","Requirement already satisfied: charset-normalizer<3,>=2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.1.1)\n","Requirement already satisfied: blis<0.8.0,>=0.7.8 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from thinc<8.2.0,>=8.1.8->spacy) (0.7.9)\n","Requirement already satisfied: confection<1.0.0,>=0.0.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from thinc<8.2.0,>=8.1.8->spacy) (0.0.4)\n","Requirement already satisfied: click<9.0.0,>=7.1.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from typer<0.8.0,>=0.3.0->spacy) (8.0.4)\n","Requirement already satisfied: MarkupSafe>=2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from jinja2->spacy) (2.1.1)\n","Collecting en-core-web-sm==3.5.0\n","\u001b[33m  WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:1125)'))': /explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl\u001b[0m\u001b[33m\n","\u001b[0m\u001b[33m  WARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:1125)'))': /explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl\u001b[0m\u001b[33m\n","\u001b[0m  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl (12.8 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0mm\n","\u001b[?25hRequirement already satisfied: spacy<3.6.0,>=3.5.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from en-core-web-sm==3.5.0) (3.5.2)\n","Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (8.1.10)\n","Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (6.3.0)\n","Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.0.12)\n","Requirement already satisfied: packaging>=20.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (23.0)\n","Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.0.8)\n","Requirement already satisfied: jinja2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.1.2)\n","Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.4.6)\n","Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.3.0)\n","Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.0.7)\n","Requirement already satisfied: typer<0.8.0,>=0.3.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.7.0)\n","Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.0.4)\n","Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.0.9)\n","Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.0.8)\n","Requirement already satisfied: numpy>=1.15.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.24.2)\n","Requirement already satisfied: pathy>=0.10.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.10.1)\n","Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.1.1)\n","Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (4.64.1)\n","Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.10.7)\n","Requirement already satisfied: setuptools in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (65.6.3)\n","Requirement already satisfied: requests<3.0.0,>=2.13.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.28.1)\n","Requirement already satisfied: typing-extensions>=4.2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (4.4.0)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.26.14)\n","Requirement already satisfied: charset-normalizer<3,>=2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.1.1)\n","Requirement already satisfied: certifi>=2017.4.17 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2022.12.7)\n","Requirement already satisfied: idna<4,>=2.5 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.4)\n","Requirement already satisfied: blis<0.8.0,>=0.7.8 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.7.9)\n","Requirement already satisfied: confection<1.0.0,>=0.0.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.0.4)\n","Requirement already satisfied: click<9.0.0,>=7.1.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from typer<0.8.0,>=0.3.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (8.0.4)\n","Requirement already satisfied: MarkupSafe>=2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from jinja2->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.1.1)\n","\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n","You can now load the package via spacy.load('en_core_web_sm')\n","Collecting zh-core-web-sm==3.5.0\n","  Downloading https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.5.0/zh_core_web_sm-3.5.0-py3-none-any.whl (48.5 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.5/48.5 MB\u001b[0m \u001b[31m974.3 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m0:01\u001b[0mm0:02\u001b[0mm\n","\u001b[?25hRequirement already satisfied: spacy-pkuseg<0.1.0,>=0.0.27 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from zh-core-web-sm==3.5.0) (0.0.32)\n","Requirement already satisfied: spacy<3.6.0,>=3.5.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from zh-core-web-sm==3.5.0) (3.5.2)\n","Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (3.0.8)\n","Requirement already satisfied: pathy>=0.10.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (0.10.1)\n","Requirement already satisfied: requests<3.0.0,>=2.13.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (2.28.1)\n","Requirement already satisfied: numpy>=1.15.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (1.24.2)\n","Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (6.3.0)\n","Requirement already satisfied: typer<0.8.0,>=0.3.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (0.7.0)\n","Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (2.0.8)\n","Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (1.1.1)\n","Requirement already satisfied: setuptools in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (65.6.3)\n","Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (3.3.0)\n","Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (3.0.12)\n","Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (4.64.1)\n","Requirement already satisfied: packaging>=20.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (23.0)\n","Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (1.0.4)\n","Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (2.4.6)\n","Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (1.0.9)\n","Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (2.0.7)\n","Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (1.10.7)\n","Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (8.1.10)\n","Requirement already satisfied: jinja2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (3.1.2)\n","Requirement already satisfied: typing-extensions>=4.2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (4.4.0)\n","Requirement already satisfied: charset-normalizer<3,>=2 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (2.1.1)\n","Requirement already satisfied: certifi>=2017.4.17 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (2022.12.7)\n","Requirement already satisfied: idna<4,>=2.5 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (3.4)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (1.26.14)\n","Requirement already satisfied: confection<1.0.0,>=0.0.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (0.0.4)\n","Requirement already satisfied: blis<0.8.0,>=0.7.8 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (0.7.9)\n","Requirement already satisfied: click<9.0.0,>=7.1.1 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from typer<0.8.0,>=0.3.0->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (8.0.4)\n","Requirement already satisfied: MarkupSafe>=2.0 in /Users/baozongbo/opt/anaconda3/lib/python3.8/site-packages (from jinja2->spacy<3.6.0,>=3.5.0->zh-core-web-sm==3.5.0) (2.1.1)\n","\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n","You can now load the package via spacy.load('zh_core_web_sm')\n"]}],"source":["#!python -m pip install --upgrade pip\n","!python -m pip install torchtext==0.6.0\n","!python -m pip install einops\n","from einops import rearrange\n","!python -m pip install spacy\n","!python -m spacy download en_core_web_sm\n","!python -m spacy download zh_core_web_sm"]},{"cell_type":"code","execution_count":61,"metadata":{},"outputs":[],"source":["from torchtext.data import Field\n","from torch.utils.data import Dataset, random_split\n","import json\n","\n","max_dataset_size = 4400\n","train_set_size = 4000\n","valid_set_size = 400\n","\n","max_input_length = 128\n","max_target_length = 128\n","EPOCH = 30\n","\n","SRC = Field(tokenize = \"spacy\",\n","            tokenizer_language=\"zh_core_web_sm\",\n","            init_token = '<sos>',\n","            eos_token = '<eos>',\n","            lower = True,\n","            batch_first=True)\n","            #truncate_first=True,\n","            #fix_length=max_input_length)\n","\n","TRG = Field(tokenize = \"spacy\",\n","            tokenizer_language=\"en_core_web_sm\",\n","            init_token = '<sos>',\n","            eos_token = '<eos>',\n","            lower = True,\n","            batch_first=True)\n","            #truncate_first=True,\n","            #fix_length=max_target_length)\n","\n","\n","zh_words_set, en_words_set = [[]], [[]]\n","\n","class TRANS(Dataset):\n","    def __init__(self, data_file):\n","        self.en_data, self.zh_data = self.load_data(data_file)\n","    \n","    def load_data(self, data_file):\n","        en_data, zh_data = [], []\n","        with open(data_file, 'rt', encoding='utf-8') as f:\n","            for idx, line in enumerate(f):\n","                if idx >= max_dataset_size:\n","                    break\n","                sample = json.loads(line.strip()) #Sample: dict()\n","\n","                en_data.append(TRG.tokenize(sample['english']))\n","\n","                zh_data.append(SRC.tokenize(sample['chinese']))\n","\n","        return en_data, zh_data #A dict(idx): EN, ZH\n","    \n","    def __len__(self):\n","        return len(self.en_data)\n","\n","    def __getitem__(self, idx):\n","        return self.en_data[idx], self.zh_data[idx]\n","\n","data = TRANS('data/translation2019zh_train.json')\n","train_data, valid_data = random_split(data, [train_set_size, valid_set_size])\n","test_data = TRANS('data/translation2019zh_valid.json')\n","\n","SRC.build_vocab(data.zh_data, min_freq=3)\n","TRG.build_vocab(data.en_data, min_freq=3)"]},{"cell_type":"code","execution_count":66,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["4254\n","3908\n","['<unk>', '<pad>', '<sos>', '<eos>', 'the']\n","634\n","pad index: 1\n","sos index: 2\n","eos index: 3\n","unk index: 0\n"]}],"source":["print(len(TRG.vocab))\n","print(len(SRC.vocab))\n","# Printing a list of tokens mapping integer to strings\n","print(TRG.vocab.itos[:5])\n","# Printing a dict mapping tokens to indices\n","#print(TRG.vocab.stoi)\n","# Printing the index of an actual word\n","print(SRC.vocab.stoi['游戏'])\n","# Checking index of special tokens\n","import torch.nn as nn\n","PAD_IDX = TRG.vocab.stoi['<pad>']\n","SOS_IDX = TRG.vocab.stoi['<sos>']\n","EOS_IDX = TRG.vocab.stoi['<eos>']\n","UNK_IDX = TRG.vocab.stoi['<unk>']\n","print('pad index:', PAD_IDX)\n","print('sos index:', SOS_IDX)\n","print('eos index:', EOS_IDX)\n","print('unk index:', UNK_IDX)\n","\n","# criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX)"]},{"cell_type":"code","execution_count":84,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Using cpu device\n"]}],"source":["import torch\n","from torch.utils.data import DataLoader\n","#torch.cuda.empty_cache() #清空缓存\n","\n","\n","device = 'cuda' if torch.cuda.is_available() else 'cpu'\n","print(f'Using {device} device')\n","\n","def collote_fn(batch_samples):\n","    en_batch, zh_batch = zip(*batch_samples)\n","    #print(en_batch,\"\\n\",zh_batch)\n","    #print(\"\\n\")\n","    #zh_batch = SRC.pad(zh_batch)\n","    zh_batch = SRC.process(zh_batch)\n","    #en_batch = TRG.pad(en_batch)    \n","    en_batch = TRG.process(en_batch)\n","    \n","    return zh_batch.transpose(0,1), en_batch.transpose(0,1)\n","\n","train_dataloader = DataLoader(train_data, batch_size=32, shuffle=True, collate_fn=collote_fn)\n","valid_dataloader = DataLoader(valid_data, batch_size=32, shuffle=False, collate_fn=collote_fn)\n","test_dataloader = DataLoader(test_data, batch_size=32, shuffle=False, collate_fn=collote_fn)"]},{"cell_type":"code","execution_count":94,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["3908\n","4254\n"]}],"source":["print(len(SRC.vocab))\n","print(len(TRG.vocab))"]},{"cell_type":"code","execution_count":95,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["tensor(3888)\n","tensor(4210)\n"]}],"source":["for x,y in train_dataloader:\n","    print(torch.max(x))\n","    print(torch.max(y))\n","    break"]},{"cell_type":"code","execution_count":117,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":9153,"status":"ok","timestamp":1683736487942,"user":{"displayName":"张耒阳","userId":"04359299908287882141"},"user_tz":-480},"id":"wSL_pS7iTOab","outputId":"cb6c2f67-ca87-48c3-86a3-a3c9094a3097"},"outputs":[{"name":"stdout","output_type":"stream","text":["The model has 11,628,190 trainable parameters\n"]},{"name":"stderr","output_type":"stream","text":["\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n"," 16%|█▌        | 20/125 [01:02<05:28,  3.13s/it]\n"]},{"ename":"KeyboardInterrupt","evalue":"","output_type":"error","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","Cell \u001b[0;32mIn[117], line 265\u001b[0m\n\u001b[1;32m    261\u001b[0m \u001b[39mfor\u001b[39;00m epoch \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(N_EPOCHS):\n\u001b[1;32m    263\u001b[0m     start_time \u001b[39m=\u001b[39m time\u001b[39m.\u001b[39mtime()\n\u001b[0;32m--> 265\u001b[0m     train_loss \u001b[39m=\u001b[39m train(model, train_dataloader, optimizer, criterion, CLIP)\n\u001b[1;32m    266\u001b[0m     \u001b[39m# 每训练一个轮次，测试一次\u001b[39;00m\n\u001b[1;32m    267\u001b[0m     valid_loss \u001b[39m=\u001b[39m evaluate(model, valid_dataloader, criterion)\n","Cell \u001b[0;32mIn[117], line 195\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(model, iterator, optimizer, criterion, clip)\u001b[0m\n\u001b[1;32m    189\u001b[0m \u001b[39m# print(\"trg size: \", trg.shape)\u001b[39;00m\n\u001b[1;32m    190\u001b[0m \u001b[39m# trg = [(trg len - 1) * batch size]\u001b[39;00m\n\u001b[1;32m    191\u001b[0m \u001b[39m# output = [(trg len - 1) * batch size, output dim]\u001b[39;00m\n\u001b[1;32m    193\u001b[0m loss \u001b[39m=\u001b[39m criterion(output, trg) \u001b[39m# 计算损失\u001b[39;00m\n\u001b[0;32m--> 195\u001b[0m loss\u001b[39m.\u001b[39;49mbackward()  \u001b[39m# 反向传播\u001b[39;00m\n\u001b[1;32m    197\u001b[0m torch\u001b[39m.\u001b[39mnn\u001b[39m.\u001b[39mutils\u001b[39m.\u001b[39mclip_grad_norm_(model\u001b[39m.\u001b[39mparameters(), clip)\n\u001b[1;32m    199\u001b[0m optimizer\u001b[39m.\u001b[39mstep()  \u001b[39m# 更新参数\u001b[39;00m\n","File \u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[1;32m    478\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    479\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[1;32m    480\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    485\u001b[0m         inputs\u001b[39m=\u001b[39minputs,\n\u001b[1;32m    486\u001b[0m     )\n\u001b[0;32m--> 487\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\n\u001b[1;32m    488\u001b[0m     \u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs\n\u001b[1;32m    489\u001b[0m )\n","File \u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    195\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[1;32m    197\u001b[0m \u001b[39m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    198\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    199\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    201\u001b[0m     tensors, grad_tensors_, retain_graph, create_graph, inputs,\n\u001b[1;32m    202\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}],"source":["import torch.nn as nn\n","\n","\n","class Encoder(nn.Module):\n","    def __init__(self, input_dim, emb_dim, hid_dim, n_layers, dropout):\n","        super().__init__()\n","        self.hid_dim = hid_dim\n","        self.n_layers = n_layers\n","        self.embedding = nn.Embedding(input_dim, emb_dim)  # 词嵌入\n","        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout=dropout)\n","        self.dropout = nn.Dropout(dropout)\n","\n","    def forward(self, src):\n","        # src = (src len, batch size)\n","        embedded = self.dropout(self.embedding(src))\n","        # embedded = (src len, batch size, emb dim)\n","        outputs, (hidden, cell) = self.rnn(embedded)\n","        # outputs = (src len, batch size, hid dim * n directions)\n","        # hidden = (n layers * n directions, batch size, hid dim)\n","        # cell = (n layers * n directions, batch size, hid dim)\n","        # rnn的输出总是来自顶部的隐藏层\n","        return hidden, cell\n","\n","\n","class Decoder(nn.Module):\n","    def __init__(self, output_dim, emb_dim, hid_dim, n_layers, dropout):\n","        super().__init__()\n","        self.output_dim = output_dim\n","        self.hid_dim = hid_dim\n","        self.n_layers = n_layers\n","        self.embedding = nn.Embedding(output_dim, emb_dim)\n","        self.rnn = nn.LSTM(emb_dim, hid_dim, n_layers, dropout=dropout)\n","        self.fc_out = nn.Linear(hid_dim, output_dim)\n","        self.dropout = nn.Dropout(dropout)\n","\n","    def forward(self, input, hidden, cell):\n","        # 各输入的形状\n","        # input = (batch size)\n","        # hidden = (n layers * n directions, batch size, hid dim)\n","        # cell = (n layers * n directions, batch size, hid dim)\n","\n","        # LSTM是单向的  ==> n directions == 1\n","        # hidden = (n layers, batch size, hid dim)\n","        # cell = (n layers, batch size, hid dim)\n","\n","        input = input.unsqueeze(0)  # (batch size)  --> [1, batch size)\n","\n","        embedded = self.dropout(self.embedding(input))  # (1, batch size, emb dim)\n","\n","        output, (hidden, cell) = self.rnn(embedded, (hidden, cell))\n","        # LSTM理论上的输出形状\n","        # output = (seq len, batch size, hid dim * n directions)\n","        # hidden = (n layers * n directions, batch size, hid dim)\n","        # cell = (n layers * n directions, batch size, hid dim)\n","\n","        # 解码器中的序列长度 seq len == 1\n","        # 解码器的LSTM是单向的 n directions == 1 则实际上\n","        # output = (1, batch size, hid dim)\n","        # hidden = (n layers, batch size, hid dim)\n","        # cell = (n layers, batch size, hid dim)\n","\n","        prediction = self.fc_out(output.squeeze(0))\n","\n","        # prediction = (batch size, output dim)\n","\n","        return prediction, hidden, cell\n","\n","\n","class Seq2Seq(nn.Module):\n","    def __init__(self, input_word_count, output_word_count, encode_dim, decode_dim, hidden_dim, n_layers,\n","                 encode_dropout, decode_dropout, device):\n","        \"\"\"\n","\n","        :param input_word_count:    英文词表的长度     34737\n","        :param output_word_count:   中文词表的长度     4015\n","        :param encode_dim:          编码器的词嵌入维度\n","        :param decode_dim:          解码器的词嵌入维度\n","        :param hidden_dim:          LSTM的隐藏层维度\n","        :param n_layers:            采用n层LSTM\n","        :param encode_dropout:      编码器的dropout概率\n","        :param decode_dropout:      编码器的dropout概率\n","        :param device:              cuda / cpu\n","        \"\"\"\n","        super().__init__()\n","        self.encoder = Encoder(input_word_count, encode_dim, hidden_dim, n_layers, encode_dropout)\n","        self.decoder = Decoder(output_word_count, decode_dim, hidden_dim, n_layers, decode_dropout)\n","        self.device = device\n","\n","    def forward(self, src, trg, teacher_forcing_ratio=0.5):\n","        # src = (src len, batch size)\n","        # trg = (trg len, batch size)\n","        # teacher_forcing_ratio 定义使用Teacher Forcing的比例\n","        # 例如 if teacher_forcing_ratio is 0.75 we use ground-truth inputs 75% of the time\n","        batch_size = trg.shape[1]\n","        trg_len = trg.shape[0]\n","        trg_vocab_size = self.decoder.output_dim  # 实际上就是中文词表的长度\n","        # 初始化保存解码器输出的Tensor\n","        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n","\n","        # 编码器的隐藏层输出将作为i解码器的第一个隐藏层输入\n","        hidden, cell = self.encoder(src)\n","\n","        # 解码器的第一个输入应该是起始标识符<sos>\n","        input = trg[0, :]  # 取trg的第“0”行所有列  “0”指的是索引\n","        # 从the_collate_fn函数中可以看出trg的第“0”行全是0，也就是起始标识符对应的ID\n","\n","        for t in range(1, trg_len): # 从 trg的第\"1\"行开始遍历\n","            # 解码器的输入包括：起始标识符的词嵌入input; 编码器输出的 hidden and cell states\n","            # 解码器的输出包括：输出张量(predictions) and new hidden and cell states\n","            output, hidden, cell = self.decoder(input, hidden, cell)\n","\n","            # 保存每次预测结果于outputs\n","            # outputs (trg_len, batch_size, trg_vocab_size)\n","            # output  (batch size, trg_vocab_size)\n","            outputs[t] = output\n","\n","            # 随机决定是否使用Teacher Forcing\n","            teacher_force = random.random() < teacher_forcing_ratio\n","\n","            # output  (batch size, trg_vocab_size)  沿dim=1取最大值索引\n","            top1 = output.argmax(dim=1)  # (batch size, )\n","\n","            # if teacher forcing, 以真实值作为下一个输入 否则 使用预测值\n","            input = trg[t] if teacher_force else top1\n","\n","        return outputs\n","\n","\n","source_word_count = len(SRC.vocab)  # 中文词表的长度     \n","target_word_count = len(TRG.vocab)  # 英文词表的长度     \n","encode_dim = 256    # 编码器的词嵌入维度\n","decode_dim = 256    # 解码器的词嵌入维度\n","hidden_dim = 512    # LSTM的隐藏层维度\n","n_layers = 2        # 采用n层LSTM\n","encode_dropout = 0.5    # 编码器的dropout概率\n","decode_dropout = 0.5    # 编码器的dropout概率\n","device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')  # GPU可用 用GPU\n","# Seq2Seq模型实例化\n","model = Seq2Seq(source_word_count, target_word_count, encode_dim, decode_dim, hidden_dim, n_layers, encode_dropout,\n","                decode_dropout, device).to(device)\n","\n","# 初始化模型参数\n","def init_weights(m):\n","    for name, param in m.named_parameters():\n","        nn.init.uniform_(param.data, -0.08, 0.08)\n","\n","model.apply(init_weights)  \n","\n","# 统计Seq2Seq模型中可训练的参数个数\n","def count_parameters(model):\n","    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n","\n","print(f'The model has {count_parameters(model):,} trainable parameters')\n","\n","import torch.optim as optim\n","\n","# 定义优化器\n","optimizer = optim.Adam(model.parameters())\n","\n","# 定义损失函数\n","criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX).to(device)  # 忽略填充标识符的索引\n","from tqdm import tqdm\n","train_loss_list, val_loss_list = [],[]\n","# 训练策略\n","def train(model, iterator, optimizer, criterion, clip):\n","    model.train()  # 切换到训练模式\n","    epoch_loss = 0\n","\n","    # progress_bar = tqdm(range(len(iterator)),total=len(iterator)) #进度条\n","    # progress_bar.set_description(f'loss: {0:>7f}')\n","    for batch_idx, (src,trg) in tqdm(enumerate(iterator), total=len(iterator)):\n","        src = src.to(device)\n","        trg = trg.to(device)\n","\n","        optimizer.zero_grad()  # 梯度清零\n","        # print(\"0\")\n","        output = model(src, trg)  # 前向传播\n","        # print(\"output size: \", output.shape)\n","        # trg = [trg len, batch size]\n","        # output = [trg len, batch size, output dim]\n","\n","        output_dim = output.shape[-1]\n","        # print(\"output_dim: \", output_dim)\n","\n","        output = output[1:].reshape(-1, output_dim)\n","        # print(\"output size: \", output.shape)\n","\n","        trg = trg[1:].reshape(-1)\n","        # print(\"trg size: \", trg.shape)\n","        # trg = [(trg len - 1) * batch size]\n","        # output = [(trg len - 1) * batch size, output dim]\n","\n","        loss = criterion(output, trg) # 计算损失\n","\n","        loss.backward()  # 反向传播\n","\n","        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n","\n","        optimizer.step()  # 更新参数\n","\n","        epoch_loss += loss.item()\n","        # print(f'[TRAIN] loss: {epoch_loss/(batch_idx+1):>7f}')\n","        # print(\"Prosess: \", batch_idx*32/len(iterator))\n","        # progress_bar.set_description(f'[TRAIN] loss: {epoch_loss/(batch_idx+1):>7f}')\n","        # progress_bar.update(1) #更新进度条\n","    train_loss_list.append(epoch_loss / len(iterator))\n","    return epoch_loss / len(iterator)\n","\n","# 验证策略\n","def evaluate(model, iterator, criterion):\n","    model.eval() # 切换到验证模式\n","\n","    epoch_loss = 0\n","    # progress_bar = tqdm(range(len(iterator))) #进度条\n","    # progress_bar.set_description(f'loss: {0:>7f}')\n","    with torch.no_grad():  # 不计算梯度\n","        for batch_idx, (src,trg) in tqdm(enumerate(iterator), total=len(iterator)):\n","            src = src.to(device)\n","            trg = trg.to(device)\n","\n","            output = model(src, trg, teacher_forcing_ratio=0)  # 验证时禁用Teacher Forcing\n","\n","            # trg = [trg len, batch size]\n","            # output = [trg len, batch size, output dim]\n","\n","            output_dim = output.shape[-1]\n","\n","            output = output[1:].reshape(-1, output_dim)\n","            trg = trg[1:].reshape(-1)\n","\n","            # trg = [(trg len - 1) * batch size]\n","            # output = [(trg len - 1) * batch size, output dim]\n","\n","            loss = criterion(output, trg)\n","\n","            epoch_loss += loss.item()\n","            # print(f'[VAL] loss: {epoch_loss/(batch_idx+1):>7f}')\n","            # progress_bar.set_description(f'[VAL] loss: {epoch_loss/(batch_idx+1):>7f}')\n","            # progress_bar.update(1) #更新进度条\n","    val_loss_list.append(epoch_loss / len(iterator)) #记录当前epoch的loss\n","\n","    return epoch_loss / len(iterator)\n","\n","# 记录每个epoch的用时\n","def epoch_time(start_time, end_time):\n","    elapsed_time = end_time - start_time\n","    elapsed_mins = int(elapsed_time / 60)\n","    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n","    return elapsed_mins, elapsed_secs\n","\n","\n","import math\n","import time\n","\n","N_EPOCHS = 5  # 训练轮次\n","CLIP = 1\n","\n","best_valid_loss = float('inf')\n","\n","# 开始训练\n","for epoch in range(N_EPOCHS):\n","\n","    start_time = time.time()\n","\n","    train_loss = train(model, train_dataloader, optimizer, criterion, CLIP)\n","    # 每训练一个轮次，测试一次\n","    valid_loss = evaluate(model, valid_dataloader, criterion)\n","\n","    end_time = time.time()\n","\n","    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n","\n","    if valid_loss < best_valid_loss: # 保存最优模型(验证loss阶段性最低时)\n","        best_valid_loss = valid_loss\n","        torch.save(model.state_dict(), 'best_model.pth')\n","\t\n","\t# 打印相关指标\n","    print(f'Epoch: {epoch + 1:02} | Time: {epoch_mins}m {epoch_secs}s')\n","    print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n","    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')\n"]},{"cell_type":"code","execution_count":118,"metadata":{},"outputs":[],"source":["def string_to_indices(LANGUAGE, sentence):\n","    if (LANGUAGE==TRG):\n","        words = sentence.split()\n","    else:\n","        words = LANGUAGE.tokenize(sentence)\n","    indices = []\n","    for word in words:\n","        if word in LANGUAGE.vocab.stoi:\n","            index = LANGUAGE.vocab.stoi[word]\n","            indices.append(index)\n","        else:\n","            index = LANGUAGE.vocab.stoi['<unk>']\n","            indices.append(index)\n","    result = torch.tensor(indices)\n","    return result\n","\n","def inference(model, example_sentence_src,max_len=128):\n","    # Translate example sentence\n","    example_tensor_src = string_to_indices(SRC, example_sentence_src).view(-1, 1) #[Sentence_length, Batch]\n","    example_sentence_tgt = '<sos>' #起始字符\n","    example_tensor_tgt = string_to_indices(TRG, example_sentence_tgt).view(-1, 1) #将起始字符也转换成[Sentence_length, Batch]\n","    src = example_tensor_src.to(device)\n","    #print(src.shape)\n","    tgt = example_tensor_tgt.to(device)\n","    #print(tgt.shape)\n","    model.eval()\n","    with torch.no_grad():\n","        for i in range(max_len): #最大长度为128\n","            #print('Src:', src)\n","            #print('Tgt:', tgt)\n","\n","            # ###下面部分同正常Transforemer模型\n","            # src_key_padding_mask = src == PAD_IDX\n","            # tgt_key_padding_mask = tgt == PAD_IDX\n","            # memory_key_padding_mask = src_key_padding_mask.clone()\n","            # src_key_padding_mask = rearrange(src_key_padding_mask, 'n s -> s n')\n","            # tgt_key_padding_mask = rearrange(tgt_key_padding_mask, 'n s -> s n')\n","            # memory_key_padding_mask = rearrange(memory_key_padding_mask, 'n s -> s n')\n","            # tgt_mask = gen_nopeek_mask(tgt.shape[0]).to(device)\n","            # #print('Tgt mask:', tgt_mask)\n","\n","            # output = model(src, tgt, 0, src_key_padding_mask=src_key_padding_mask, tgt_key_padding_mask=tgt_key_padding_mask, memory_key_padding_mask=memory_key_padding_mask, tgt_mask=tgt_mask) #turn off teacher forcing\n","\n","            output = model(src, tgt, teacher_forcing_ratio=0) #turn off teacher forcing\n","\n","            # print('Output:', output.shape) \n","            # TODO: Check that the argmax line is correct\n","            output_index = torch.argmax(output, dim=2)[-1].item() #得到输出后，在vocab维度上求argmax（找到可能性最大的单词idx)\n","            #print(output_index)\n","            if (output_index==4 and example_sentence_tgt[-2]==','): #多个逗号\n","                break\n","            output_word = TRG.vocab.itos[output_index] #将单词idx转换为对应的单词\n","            example_sentence_tgt = example_sentence_tgt + ' ' + output_word #拼接单词\n","            #print('Translated sentence so far:', example_sentence_tgt)\n","            example_tensor_tgt = string_to_indices(TRG, example_sentence_tgt).view(-1, 1) #重新转换为#[Sentence_length, Batch]的tensor 作为target\n","            tgt = example_tensor_tgt.to(device)\n","            if output_word == '<eos>':\n","                break\n","    return example_sentence_tgt"]},{"cell_type":"code","execution_count":121,"metadata":{},"outputs":[{"data":{"text/plain":["'<sos> <unk> <unk> <unk> <unk> <unk> <unk> <eos>'"]},"execution_count":121,"metadata":{},"output_type":"execute_result"}],"source":["inference(model,\"牛逼啊，这个模型\")"]},{"cell_type":"code","execution_count":59,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":8,"status":"ok","timestamp":1683738079379,"user":{"displayName":"张耒阳","userId":"04359299908287882141"},"user_tz":-480},"id":"ok3Wc_qmGYG3","outputId":"0d651ba7-a301-4e75-9e7e-3f0ef493e960"},"outputs":[{"name":"stdout","output_type":"stream","text":["torch.Size([38, 16])\n","-------tokens---------\n","torch.Size([38])\n","------words-----\n","[['<sos>'], ['forest'], ['incarcerate'], ['unrecognized'], ['$70'], ['polypropylene'], ['licence'], ['board'], ['channel'], ['ancestry'], ['incarcerate'], ['bra'], ['phaethon'], ['trouser'], ['$70'], ['outings'], ['merchandising'], ['exploded'], ['osmotic'], ['chem'], ['<>'], ['channel'], [], ['percutaneous'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<pad>'], ['<eos>']]\n"]},{"data":{"text/plain":["(tensor([[-2.4829,  2.5140, -1.2461,  ..., -2.4494, -2.6127, -1.9710],\n","         [-2.4829,  2.5140, -1.2461,  ..., -2.4494, -2.6127, -1.9710],\n","         [-2.4830,  2.5139, -1.2462,  ..., -2.4493, -2.6127, -1.9710],\n","         ...,\n","         [-3.0057,  7.6327, -1.5604,  ..., -3.3757, -3.3229, -2.9270],\n","         [-3.0055,  7.6330, -1.5603,  ..., -3.3754, -3.3227, -2.9269],\n","         [-3.0056,  7.6328, -1.5604,  ..., -3.3756, -3.3228, -2.9270]]),\n"," tensor([ 1130,  5869, 13480,  8104,  8104, 11684,   744,  4138, 12173, 23488,\n","         19626, 14735,  8104,  8104,  8104,  8104,  6026, 16329, 19136, 23933,\n","         16384, 14922,  3839, 14712, 14446,  3401, 15383, 19136, 24320, 23391,\n","         20774, 15517,  4753,  9971, 18439, 12815, 23994, 25695,  8104,  6660,\n","          5958, 25242,  8104, 18439, 21301, 23356, 12815, 21127,  7138, 17217,\n","          8104, 21717,  6714,  9896,  2391, 14269,  2672,  5046,  1266, 14694,\n","          5961, 23994, 19326,  8471, 11089, 10513, 16245, 23994,  9534,  8104,\n","            76,  1321,  6660,  1756,  9971,  6714,  4081, 13480, 23142, 12460,\n","          8571, 26331,  5145,   513, 13480, 17284, 12815, 24203, 15212,  8324,\n","         12599, 25291, 12023,  8104,  8104,   938, 13803, 24159, 17490, 23488,\n","         21702, 12815,  6714,  5046, 20666, 18199,  6714, 20551,  3779,   405,\n","         11074,  5257,  5046, 23165, 20864, 11038,  8377,  8104, 11184, 13480,\n","         22478, 22277,  5169,  6660,  5559,  4383, 21670, 26528, 21159,  5046,\n","         23488,    35,  5394, 18954,  6660,  7537,  6714,  6714,  2213, 10039,\n","         12524,  3084,  6026, 12460, 13480,  8104,  3401,     2, 11159,  9971,\n","         14119, 26603, 23458,  4193,  6660,  5878,  8104, 23488, 12513,  9138,\n","         25399,  8280, 14484,     2, 18493,  8104, 26383, 13285,  6660,  6660,\n","         23654,  6660,  5542, 25630, 23488,  6660,  1928,   475,  9971,     2,\n","         16441, 13750,  6660, 11155, 10682, 22133,  1533, 15784, 12815,   931,\n","          8104, 11550,  8634,  6660,     2,     2,  9534,  5257, 13161, 23815,\n","         22807,  3779,  6714, 14461,  1068,  2840, 21478,  6573, 16294, 14735,\n","             2,     2,  2457, 20551,  8634, 23488,  6660, 14735,  6665, 19136,\n","         19136, 12762,  3779, 22277,  5257,  9646,     2,     2, 17355,  8634,\n","           649,   832,  6714,  9971, 11084, 22832, 18439,  6360,  8104,  8104,\n","         13111, 19136,     2,     2, 14462, 24962,  9971,  6660, 23263,     2,\n","         24365, 13477, 20788, 18493,  9027, 13160,  5046, 22832,     2,     2,\n","          9971, 26733,     2, 21862, 11026,     2,  8324, 12173, 25114, 26331,\n","          5046,  9971,  8377,  6065,     2,     2,     2, 19964,     2,  5766,\n","         18812,     2,  1986,  9971,  8557, 22939, 19196, 22314,  8104,  5128,\n","             2,     2,     2, 12524,     2, 10282,  7262,     2, 23488,     2,\n","          9971, 15956,  9971, 24093, 12476, 26638,     2,     2,     2,  6660,\n","             2, 17900,  9971,     2, 18615,     2,     2,  9971,     2,  4539,\n","         25254,  6660,     2,     2,     2, 23815,     2,  5986, 24093,     2,\n","          6866,     2,     2,     2,     2, 18396, 12815, 24093,     2,     2,\n","             2, 24365,     2, 25179, 21511,     2, 26200,     2,     2,     2,\n","             2,  9282, 21551,   121,     2,     2,     2,  5257,     2,  8104,\n","         19136,     2,  9971,     2,     2,     2,     2, 14694,  3707,  9971,\n","             2,     2,     2,  8571,     2,  7899, 22832,     2,     2,     2,\n","             2,     2,     2, 10703,  9971,     2,     2,     2,     2,  8104,\n","             2, 12815, 17512,     2,     2,     2,     2,     2,     2, 24365,\n","             2,     2,     2,     2,     2, 18092,     2, 26331,  5046,     2,\n","             2,     2,     2,     2,     2, 13460,     2,     2,     2,     2,\n","             2, 12815,     2, 12130,  4793,     2,     2,     2,     2,     2,\n","             2, 26188,     2,     2,     2,     2,     2, 11684,     2, 23697,\n","         26331,     2,     2,     2,     2,     2,     2,  9971,     2,     2,\n","             2,     2,     2, 14922,     2,  9971,   726,     2,     2,     2,\n","             2,     2,     2,     2,     2,     2,     2,     2,     2, 25695,\n","             2,     2,  3978,     2,     2,     2,     2,     2,     2,     2,\n","             2,     2,     2,     2,     2,  9971,     2,     2,  7262,     2,\n","             2,     2,     2,     2,     2,     2,     2,     2,     2,     2,\n","             2,     2,     2,     2,  9971,     2,     2,     2,     2,     2,\n","             2,     2,     1,     1,     1,     1,     1,     1,     1,     1,\n","             1,     1,     1,     1,     1,     1,     1,     1]))"]},"execution_count":59,"metadata":{},"output_type":"execute_result"}],"source":["# def inference(model, iterator):\n","#     model.eval() # 切换到验证模式\n","\n","#     epoch_loss = 0\n","\n","#     with torch.no_grad():  # 不计算梯度\n","#         for i, batch in enumerate(iterator):\n","#             if i >= 1:\n","#                 break\n","#             src = batch[0].to(device)\n","#             # print(src.shape)\n","#             # print('-------tokens---------')\n","\n","#             # test_src = src[:, 0]\n","#             # print(test_src.shape)\n","\n","#             # print('------words-----')\n","#             # print(convert_tensor(test_src))\n","\n","#             trg = batch[1].to(device)\n","\n","#             output = model(src, trg, teacher_forcing_ratio=0)  # 验证时禁用Teacher Forcing\n","\n","#             # trg = [trg len, batch size]\n","#             # output = [trg len, batch size, output dim]\n","\n","#             output_dim = output.shape[-1]\n","\n","#             output = output[1:].view(-1, output_dim)\n","#             trg = trg[1:].view(-1)\n","\n","#     return output, trg\n","\n","# def get_key (dict, value):\n","#     # search the word according the word index\n","#     return [k for k, v in dict.items() if v == value]\n","\n","# def convert_tensor(tokens):\n","#     return [get_key(en2id, token.item()) for token in tokens]\n","\n","# inference(model, train_data_loader)"]}],"metadata":{"accelerator":"GPU","colab":{"authorship_tag":"ABX9TyM5r2GSTgdrpNRh6D5IfVmv","gpuType":"V100","machine_shape":"hm","mount_file_id":"1jGSYtNCizFifHOAvbpaMkRIPbaAVgtmt","provenance":[]},"gpuClass":"standard","kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.8"}},"nbformat":4,"nbformat_minor":0}