yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-09-26 16:52:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenizing example

Browse Source
This commit is contained in:
rasbt 2024-05-06 07:16:40 -05:00
parent 15d6f29cf8
commit 78829f28e9
1 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ch06/01_main-chapter-code/ch06.ipynb
View File

@ -172,7 +172,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"File downloaded and saved as sms_spam_collection/SMSSpamCollection.tsv\n"
"sms_spam_collection/SMSSpamCollection.tsv already exists. Skipping download and extraction.\n"
]
}
],
@ -557,18 +557,16 @@
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[42, 13, 314, 481, 1908, 340, 757]"
"name": "stdout",
"output_type": "stream",
"text": [
"[1212, 318, 262, 717, 2420, 3275]\n"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tokenizer.encode(\"K. I will sent it again\")"
"token_ids = tokenizer.encode(\"This is the first text message\")\n",
"print(token_ids)"
]
},
{