From 485bc7d7d65c1cc7cb75ffd17f50e05f51538b72 Mon Sep 17 00:00:00 2001
From: Kevin Hu <kevinhu.sh@gmail.com>
Date: Fri, 14 Mar 2025 17:10:38 +0800
Subject: [PATCH] Fix: limit the depth of DFS (#6101)

### What problem does this PR solve?

#6085

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
---
 rag/nlp/rag_tokenizer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/rag/nlp/rag_tokenizer.py b/rag/nlp/rag_tokenizer.py
index c262ca9b4..6f2686851 100644
--- a/rag/nlp/rag_tokenizer.py
+++ b/rag/nlp/rag_tokenizer.py
@@ -118,6 +118,8 @@ class RagTokenizer:
 
     def dfs_(self, chars, s, preTks, tkslist):
         res = s
+        if len(tkslist) >= 2048:
+            return res
         # if s > MAX_L or s>= len(chars):
         if s >= len(chars):
             tkslist.append(preTks)