yujunjun/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2025-08-29 19:10:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

basepath

Browse Source
This commit is contained in:
rasbt 2024-05-12 09:27:38 -05:00
parent 913662ebeb
commit a0adf0d5d3
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ch06/03_bonus_imdb-classification/train-bert-hf.py
View File

@ -236,6 +236,8 @@ if __name__ == "__main__":
pad_token_id = tokenizer.encode(tokenizer.pad_token) pad_token_id = tokenizer.encode(tokenizer.pad_token)
base_path = "."
train_dataset = IMDBDataset(base_path / "train.csv", max_length=256, tokenizer=tokenizer, pad_token_id=pad_token_id) train_dataset = IMDBDataset(base_path / "train.csv", max_length=256, tokenizer=tokenizer, pad_token_id=pad_token_id)
val_dataset = IMDBDataset(base_path / "val.csv", max_length=256, tokenizer=tokenizer, pad_token_id=pad_token_id) val_dataset = IMDBDataset(base_path / "val.csv", max_length=256, tokenizer=tokenizer, pad_token_id=pad_token_id)
test_dataset = IMDBDataset(base_path / "test.csv", max_length=256, tokenizer=tokenizer, pad_token_id=pad_token_id) test_dataset = IMDBDataset(base_path / "test.csv", max_length=256, tokenizer=tokenizer, pad_token_id=pad_token_id)