yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-11-04 12:03:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update single_url.ts

Browse Source
This commit is contained in:
rafaelsideguide 2024-08-12 16:40:31 -03:00
parent bbed6ef23d
commit c3aeed510b
1 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -24,13 +24,15 @@ import { clientSideError } from "../../strings";
dotenv.config();
const useDatabaseAuth = process.env.USE_DB_AUTHENTICATION === "true";
const useScrapingBee = process.env.SCRAPING_BEE_API_KEY !== '' && process.env.SCRAPING_BEE_API_KEY !== undefined;
const useFireEngine = process.env.FIRE_ENGINE_BETA_URL !== '' && process.env.FIRE_ENGINE_BETA_URL !== undefined;
export const baseScrapers = [
"fire-engine",
"fire-engine;chrome-cdp",
"scrapingBee",
useFireEngine ? "fire-engine" : undefined,
useFireEngine ? "fire-engine;chrome-cdp" : undefined,
useScrapingBee ? "scrapingBee" : undefined,
useDatabaseAuth ? undefined : "playwright",
"scrapingBeeLoad",
useScrapingBee ? "scrapingBeeLoad" : undefined,
"fetch",
].filter(Boolean);
@ -87,11 +89,11 @@ function getScrapingFallbackOrder(
});
let defaultOrder = [
!useDatabaseAuth ? undefined : "fire-engine",
!useDatabaseAuth ? undefined : "fire-engine;chrome-cdp",
"scrapingBee",
useFireEngine ? "fire-engine" : undefined,
useFireEngine ? "fire-engine;chrome-cdp" : undefined,
useScrapingBee ? "scrapingBee" : undefined,
useScrapingBee ? "scrapingBeeLoad" : undefined,
useDatabaseAuth ? undefined : "playwright",
"scrapingBeeLoad",
"fetch",
].filter(Boolean);