yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-10-31 01:54:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(fetch): don't time out (for smart scrape)

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-15 21:57:05 -07:00
parent 0abe60085b
commit 13bd50ad2b
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/scraper/scrapeURL/lib/fetch.ts
View File

@ -4,6 +4,7 @@ import * as Sentry from "@sentry/node";
import { MockState, saveMock } from "./mock"; import { MockState, saveMock } from "./mock";
import { TimeoutSignal } from "../../../controllers/v1/types"; import { TimeoutSignal } from "../../../controllers/v1/types";
import { fireEngineURL } from "../engines/fire-engine/scrape"; import { fireEngineURL } from "../engines/fire-engine/scrape";
import { fetch, RequestInit, Response, FormData, Agent } from "undici";
export type RobustFetchParams<Schema extends z.Schema<any>> = { export type RobustFetchParams<Schema extends z.Schema<any>> = {
url: string; url: string;
@ -78,6 +79,10 @@ export async function robustFetch<
...(headers !== undefined ? headers : {}), ...(headers !== undefined ? headers : {}),
}, },
signal: abort, signal: abort,
dispatcher: new Agent({
headersTimeout: 0,
bodyTimeout: 0,
}),
...(body instanceof FormData ...(body instanceof FormData
? { ? {
body, body,