yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-08-28 10:30:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove redundant code lines (#247)

Browse Source
This commit is contained in:
Sebastian Raschka 2024-06-25 21:44:19 -05:00 committed by GitHub
parent f46441d53f
commit 4fef19e016
1 changed files with 7 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
ch02/01_main-chapter-code/ch02.ipynb
View File

@ -785,9 +785,6 @@
"metadata": {},
"outputs": [],
"source": [
"preprocessed = re.split(r'([,.?_!\"()\\']|--|\\s)', raw_text)\n",
"preprocessed = [item.strip() for item in preprocessed if item.strip()]\n",
"\n",
"all_tokens = sorted(list(set(preprocessed)))\n",
"all_tokens.extend([\"<|endoftext|>\", \"<|unk|>\"])\n",
"\n",
@ -803,7 +800,7 @@
{
"data": {
"text/plain": [
"1161"
"1132"
]
},
"execution_count": 19,
@ -825,11 +822,11 @@
"name": "stdout",
"output_type": "stream",
"text": [
"('younger', 1156)\n",
"('your', 1157)\n",
"('yourself', 1158)\n",
"('<|endoftext|>', 1159)\n",
"('<|unk|>', 1160)\n"
"('younger', 1127)\n",
"('your', 1128)\n",
"('yourself', 1129)\n",
"('<|endoftext|>', 1130)\n",
"('<|unk|>', 1131)\n"
]
}
],
@ -918,22 +915,7 @@
{
"data": {
"text/plain": [
"[1160,\n",
" 5,\n",
" 362,\n",
" 1155,\n",
" 642,\n",
" 1000,\n",
" 10,\n",
" 1159,\n",
" 57,\n",
" 1013,\n",
" 981,\n",
" 1009,\n",
" 738,\n",
" 1013,\n",
" 1160,\n",
" 7]"
"[1131, 5, 355, 1126, 628, 975, 10, 1130, 55, 988, 956, 984, 722, 988, 1131, 7]"
]
},
"execution_count": 23,