yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-12-30 00:37:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(scrapeURL): reintroduce default timeout for simple queries (#1440)

Browse Source 
* feat(scrapeURL): reintroduce default timeout

* fix

* adjust timeouts
This commit is contained in:
Gergő Móricz 2025-04-10 12:29:54 +02:00 committed by GitHub
parent 7bb5643028
commit 4294face78
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/__tests__/snips/batch-scrape.test.ts
View File

@ -8,7 +8,7 @@ describe("Batch scrape tests", () => {
expect(response.body.data[0]).toHaveProperty("markdown");
expect(response.body.data[0].markdown).toContain("Firecrawl");
}, 30000);
}, 180000);
if (!process.env.TEST_SUITE_SELF_HOSTED) {
describe("JSON format", () => {
@ -45,7 +45,7 @@ describe("Batch scrape tests", () => {
expect(response.body.data[0].json).toHaveProperty("is_open_source");
expect(response.body.data[0].json.is_open_source).toBe(true);
expect(typeof response.body.data[0].json.is_open_source).toBe("boolean");
}, 30000);
}, 180000);
});
}
});

4
apps/api/src/scraper/scrapeURL/index.ts
View File

@ -230,7 +230,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
const timeToRun =
meta.options.timeout !== undefined
? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))
: undefined;
: (!meta.options.actions && !meta.options.jsonOptions && !meta.options.extract)
? Math.round(120000 / Math.min(fallbackList.length, 2))
: undefined;
for (const { engine, unsupportedFeatures } of fallbackList) {
meta.internalOptions.abort?.throwIfAborted();