yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-11-09 14:57:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update single_url.ts

Browse Source
This commit is contained in:
rafaelsideguide 2024-08-12 16:40:31 -03:00
parent bbed6ef23d
commit c3aeed510b
1 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -24,13 +24,15 @@ import { clientSideError } from "../../strings";
dotenv.config(); dotenv.config();
const useDatabaseAuth = process.env.USE_DB_AUTHENTICATION === "true"; const useDatabaseAuth = process.env.USE_DB_AUTHENTICATION === "true";
const useScrapingBee = process.env.SCRAPING_BEE_API_KEY !== '' && process.env.SCRAPING_BEE_API_KEY !== undefined;
const useFireEngine = process.env.FIRE_ENGINE_BETA_URL !== '' && process.env.FIRE_ENGINE_BETA_URL !== undefined;
export const baseScrapers = [ export const baseScrapers = [
"fire-engine", useFireEngine ? "fire-engine" : undefined,
"fire-engine;chrome-cdp", useFireEngine ? "fire-engine;chrome-cdp" : undefined,
"scrapingBee", useScrapingBee ? "scrapingBee" : undefined,
useDatabaseAuth ? undefined : "playwright", useDatabaseAuth ? undefined : "playwright",
"scrapingBeeLoad", useScrapingBee ? "scrapingBeeLoad" : undefined,
"fetch", "fetch",
].filter(Boolean); ].filter(Boolean);
@ -87,11 +89,11 @@ function getScrapingFallbackOrder(
}); });
let defaultOrder = [ let defaultOrder = [
!useDatabaseAuth ? undefined : "fire-engine", useFireEngine ? "fire-engine" : undefined,
!useDatabaseAuth ? undefined : "fire-engine;chrome-cdp", useFireEngine ? "fire-engine;chrome-cdp" : undefined,
"scrapingBee", useScrapingBee ? "scrapingBee" : undefined,
useScrapingBee ? "scrapingBeeLoad" : undefined,
useDatabaseAuth ? undefined : "playwright", useDatabaseAuth ? undefined : "playwright",
"scrapingBeeLoad",
"fetch", "fetch",
].filter(Boolean); ].filter(Boolean);