2024-08-27 15:08:29 -03:00
|
|
|
import FirecrawlApp, { ScrapeResponse } from './firecrawl/src/index' //'@mendable/firecrawl-js';
|
2024-08-08 11:41:13 -03:00
|
|
|
import { CrawlStatusResponse } from './firecrawl/src/index';
|
2024-05-09 10:36:56 -03:00
|
|
|
|
|
|
|
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
|
|
|
|
|
|
|
|
// Scrape a website:
|
2024-08-27 15:08:29 -03:00
|
|
|
const scrapeResult = await app.scrapeUrl('firecrawl.dev') as ScrapeResponse;
|
2024-06-18 11:43:06 -03:00
|
|
|
|
2024-08-27 15:08:29 -03:00
|
|
|
if (scrapeResult) {
|
|
|
|
console.log(scrapeResult.markdown)
|
2024-06-18 11:43:06 -03:00
|
|
|
}
|
2024-05-09 10:36:56 -03:00
|
|
|
|
|
|
|
// Crawl a website:
|
|
|
|
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}, false);
|
|
|
|
console.log(crawlResult)
|
|
|
|
|
|
|
|
const jobId: string = await crawlResult['jobId'];
|
|
|
|
console.log(jobId);
|
|
|
|
|
2024-08-08 11:41:13 -03:00
|
|
|
let job: CrawlStatusResponse;
|
2024-05-09 10:36:56 -03:00
|
|
|
while (true) {
|
2024-08-08 11:41:13 -03:00
|
|
|
job = await app.checkCrawlStatus(jobId) as CrawlStatusResponse;
|
2024-05-09 10:36:56 -03:00
|
|
|
if (job.status === 'completed') {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
|
|
|
|
}
|
|
|
|
|
2024-06-18 11:43:06 -03:00
|
|
|
if (job.data) {
|
2024-08-08 11:41:13 -03:00
|
|
|
console.log(job.data[0].markdown);
|
2024-06-18 11:43:06 -03:00
|
|
|
}
|
|
|
|
|
2024-08-27 15:08:29 -03:00
|
|
|
const mapResult = await app.mapUrl('https://firecrawl.dev');
|
2024-08-08 11:41:13 -03:00
|
|
|
console.log(mapResult)
|