diff --git a/pyproject.toml b/pyproject.toml index c7fe1fb..5c9b74a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ authors = [ requires-python = ">=3.8" dependencies = [ "cached-path", + "smart_open", "pypdf", "pymupdf", "pypdfium2", @@ -52,13 +53,19 @@ dev = [ "sphinx-copybutton==0.5.2", "sphinx-autobuild==2021.3.14", "sphinx-autodoc-typehints==1.23.3", - "packaging" + "packaging", + "necessary", ] -[project.optional-dependencies] train = [ "torch", "torchvision", + "accelerate", + "datasets", + "peft", + "wandb", + "omegaconf", + "s3fs", "transformers @ git+https://github.com/huggingface/transformers.git@f9b44097261270377199bb0fcbceb01fd9f7d0c0" ]