yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-12-29 16:16:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

FIR-2006: Fix maxUrls and timeLimit parameters in Deep Research API (#1569)

Browse Source 
* FIR-2006: Fix maxUrls and timeLimit enforcement in Deep Research API

Co-Authored-By: Nicolas Camara <nicolascamara29@gmail.com>

* FIR-2006: Add tests for maxUrls and timeLimit enforcement

Co-Authored-By: Nicolas Camara <nicolascamara29@gmail.com>

* FIR-2006: Replace mocked tests with end-to-end tests for deep research

Co-Authored-By: Nicolas Camara <nicolascamara29@gmail.com>

* Delete apps/api/src/__tests__/snips/deep-research-service.test.ts

* Delete apps/api/src/__tests__/snips/lib.ts

* Revert "Delete apps/api/src/__tests__/snips/lib.ts"

This reverts commit a2af9baff89d64adc1930ea5b37b4f07f0735a67.

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Nicolas Camara <nicolascamara29@gmail.com>
This commit is contained in:
devin-ai-integration[bot] 2025-05-20 18:39:56 -03:00 committed by GitHub
parent 513f469b0f
commit 9949403b59
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 111 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
apps/api/src/__tests__/snips/lib.ts
View File

@ -276,3 +276,64 @@ export async function tokenUsage(): Promise<{ remaining_tokens: number }> {
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")).body.data;
}
// =========================================
// =========================================
async function deepResearchStart(body: {
query?: string;
maxDepth?: number;
maxUrls?: number;
timeLimit?: number;
analysisPrompt?: string;
systemPrompt?: string;
formats?: string[];
topic?: string;
jsonOptions?: any;
}) {
return await request(TEST_URL)
.post("/v1/deep-research")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(body);
}
async function deepResearchStatus(id: string) {
return await request(TEST_URL)
.get("/v1/deep-research/" + encodeURIComponent(id))
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.send();
}
function expectDeepResearchStartToSucceed(response: Awaited<ReturnType<typeof deepResearchStart>>) {
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
expect(typeof response.body.id).toBe("string");
}
export async function deepResearch(body: {
query?: string;
maxDepth?: number;
maxUrls?: number;
timeLimit?: number;
analysisPrompt?: string;
systemPrompt?: string;
formats?: string[];
topic?: string;
jsonOptions?: any;
}) {
const ds = await deepResearchStart(body);
expectDeepResearchStartToSucceed(ds);
let x;
do {
x = await deepResearchStatus(ds.body.id);
expect(x.statusCode).toBe(200);
expect(typeof x.body.status).toBe("string");
} while (x.body.status === "processing");
expect(x.body.success).toBe(true);
expect(x.body.status).toBe("completed");
return x.body;
}

56
apps/api/src/lib/deep-research/deep-research-service.ts
View File

@ -47,11 +47,29 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
const acuc = await getACUCTeam(teamId);
const checkTimeLimit = () => {
const timeElapsed = Date.now() - startTime;
const isLimitReached = timeElapsed >= timeLimit * 1000;
if (isLimitReached) {
logger.debug("[Deep Research] Time limit reached", {
timeElapsed: timeElapsed / 1000,
timeLimit
});
}
return isLimitReached;
};
try {
while (!state.hasReachedMaxDepth() && urlsAnalyzed < maxUrls) {
logger.debug("[Deep Research] Current depth:", state.getCurrentDepth());
const timeElapsed = Date.now() - startTime;
if (timeElapsed >= timeLimit * 1000) {
logger.debug("[Deep Research] URL analysis count:", {
urlsAnalyzed,
maxUrls,
timeElapsed: (Date.now() - startTime) / 1000,
timeLimit
});
if (checkTimeLimit()) {
logger.debug("[Deep Research] Time limit reached, stopping research");
break;
}
@ -142,15 +160,25 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
}
// Filter out already seen URLs and track new ones
const newSearchResults = searchResults.filter(async (result) => {
const newSearchResults: typeof searchResults = [];
for (const result of searchResults) {
if (!result.url || state.hasSeenUrl(result.url)) {
return false;
continue;
}
state.addSeenUrl(result.url);
urlsAnalyzed++;
return true;
});
if (urlsAnalyzed >= maxUrls) {
logger.debug("[Deep Research] Max URLs limit reached", { urlsAnalyzed, maxUrls });
break;
}
newSearchResults.push(result);
}
if (checkTimeLimit()) {
logger.debug("[Deep Research] Time limit reached during URL filtering");
break;
}
await state.addSources(newSearchResults.map((result) => ({
url: result.url ?? "",
@ -205,6 +233,11 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
const timeRemaining = timeLimit * 1000 - (Date.now() - startTime);
logger.debug("[Deep Research] Time remaining (ms):", { timeRemaining });
if (checkTimeLimit()) {
logger.debug("[Deep Research] Time limit reached before analysis");
break;
}
const analysis = await llmService.analyzeAndPlan(
state.getFindings(),
currentTopic,
@ -212,6 +245,11 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
options.systemPrompt ?? "",
costTracking,
);
if (checkTimeLimit()) {
logger.debug("[Deep Research] Time limit reached after analysis");
break;
}
if (!analysis) {
logger.debug("[Deep Research] Analysis failed");
@ -258,6 +296,12 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
// Final synthesis
logger.debug("[Deep Research] Starting final synthesis");
// Check time limit before final synthesis
if (checkTimeLimit()) {
logger.debug("[Deep Research] Time limit reached before final synthesis");
}
await state.addActivity([{
type: "synthesis",
status: "processing",