From 01dbc7b4733e88efd6c1e85930c707009a2a966e Mon Sep 17 00:00:00 2001
From: Christine Straub <christinemstraub@gmail.com>
Date: Tue, 20 Aug 2024 11:56:59 -0700
Subject: [PATCH] fix: `nltk` data download path to prevent redundant nested
 directories (#3546)

Closes #3543.

### Summary
This PR addresses an issue with the NLTK data download process.
Previously, when downloading NLTK data, a nested "nltk_data" directory
was created within the parent "nltk_data" directory if the parent
directory already existed. This redundant directory structure led to two
significant problems:
- errors in checking if data had already been downloaded, potentially
causing redundant downloads in subsequent calls.
- failures in loading models from the downloaded NLTK data due to
incorrect path resolution.

This fix modifies the NLTK data download logic to prevent creation of
unnecessary nested directories. If the download path ends with
"nltk_data" and that directory already exists, we now use the existing
directory instead of creating a new nested one.

### Testing
CI should pass.
---
 CHANGELOG.md                 | 11 ++++++++++-
 unstructured/__version__.py  |  2 +-
 unstructured/nlp/tokenize.py |  4 ++++
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 21c86340d..1e07fb69e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,13 @@
+## 0.15.7
+
+### Enhancements
+
+### Features
+
+### Fixes
+
+* **Fix NLTK data download path to prevent nested directories**. Resolved an issue where a nested "nltk_data" directory was created within the parent "nltk_data" directory when it already existed. This fix prevents errors in checking for existing downloads and loading models from NLTK data.
+
 ## 0.15.6
 
 ### Enhancements
@@ -10,7 +20,6 @@
 * **Update CI for `ingest-test-fixture-update-pr` to resolve NLTK model download errors.**
 * **Synchronized text and html on `TableChunk` splits.** When a `Table` element is divided during chunking to fit the chunking window, `TableChunk.text` corresponds exactly with the table text in `TableChunk.metadata.text_as_html`, `.text_as_html` is always parseable HTML, and the table is split on even row boundaries whenever possible.
 
-
 ## 0.15.5
 
 ### Enhancements
diff --git a/unstructured/__version__.py b/unstructured/__version__.py
index 986e0018a..d57a4f171 100644
--- a/unstructured/__version__.py
+++ b/unstructured/__version__.py
@@ -1 +1 @@
-__version__ = "0.15.6"  # pragma: no cover
+__version__ = "0.15.7"  # pragma: no cover
diff --git a/unstructured/nlp/tokenize.py b/unstructured/nlp/tokenize.py
index fe39e3a77..9f438ed1d 100644
--- a/unstructured/nlp/tokenize.py
+++ b/unstructured/nlp/tokenize.py
@@ -70,6 +70,10 @@ def download_nltk_packages():
     if nltk_data_dir is None:
         raise OSError("NLTK data directory does not exist or is not writable.")
 
+    # Check if the path ends with "nltk_data" and remove it if it does
+    if nltk_data_dir.endswith("nltk_data"):
+        nltk_data_dir = os.path.dirname(nltk_data_dir)
+
     def sha256_checksum(filename: str, block_size: int = 65536):
         sha256 = hashlib.sha256()
         with open(filename, "rb") as f: