feat(scrape, extract): creditsUsed, tokensUsed fields (FIR-2336) (#1683)

* fix(scrape): log FIRE-1 credits billed on failures properly

* fix dumb thinbgs

* feat(scrape, extract): creditsUsed fields

* fix(extract): call it tokensUsed

* Trigger Build

* dumb mistake, search does separate billing

apps/api/src/__tests__/snips/billing.test.ts

@@ -20,7 +20,7 @@ describe("Billing tests", () => {
             const rc1 = (await creditUsage()).remaining_credits;
             
             // Run all scrape operations in parallel with Promise.all
-            await Promise.all([
+            const [scrape1, scrape2, scrape3] = await Promise.all([
                 // scrape 1: regular fc.dev scrape (1 credit)
                 scrape({
                     url: "https://firecrawl.dev"
@@ -46,6 +46,10 @@ describe("Billing tests", () => {
                     },
                 })
             ]);
+
+            expect(scrape1.metadata.creditsUsed).toBe(1);
+            expect(scrape2.metadata.creditsUsed).toBe(1);
+            expect(scrape3.metadata.creditsUsed).toBe(5);
             
             // sum: 7 credits
 
@@ -89,6 +93,12 @@ describe("Billing tests", () => {
                     },
                 })
             ]);
+
+            expect(scrape1.data[0].metadata.creditsUsed).toBe(1);
+            expect(scrape1.data[1].metadata.creditsUsed).toBe(1);
+
+            expect(scrape2.data[0].metadata.creditsUsed).toBe(5);
+            expect(scrape2.data[1].metadata.creditsUsed).toBe(5);
             
             // sum: 12 credits
 
@@ -185,7 +195,7 @@ describe("Billing tests", () => {
         it("bills extract correctly", async () => {
             const rc1 = (await tokenUsage()).remaining_tokens;
             
-            await extract({
+            const extractResult = await extract({
                 urls: ["https://firecrawl.dev"],
                 schema: {
                     "type": "object",
@@ -201,6 +211,8 @@ describe("Billing tests", () => {
                 origin: "api-sdk",
             });
 
+            expect(extractResult.tokensUsed).toBe(305);
+
             await sleepForBatchBilling();
             
             const rc2 = (await tokenUsage()).remaining_tokens;

apps/api/src/controllers/v1/extract-status.ts

@@ -89,5 +89,6 @@ export async function extractStatusController(
     sources: extract?.showSources ? extract.sources : undefined,
     costTracking: extract?.showCostTracking ? extract.costTracking : undefined,
     sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,
+    tokensUsed: extract?.tokensBilled ? extract.tokensBilled : undefined,
   });
 }

apps/api/src/controllers/v1/types.ts

@@ -782,6 +782,7 @@ export type Document = {
     proxyUsed: "basic" | "stealth";
     cacheState?: "hit" | "miss";
     cachedAt?: string;
+    creditsUsed?: number;
     // [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
   };
   serpResults?: {
@@ -843,6 +844,7 @@ export interface ExtractResponse {
   sources?: {
     [key: string]: string[];
   };
+  tokensUsed?: number;
 }
 
 export interface ExtractResponseRequestTest {

apps/api/src/lib/extract/extract-redis.ts

@@ -40,6 +40,7 @@ export type StoredExtract = {
     [key: string]: string[];
   };
   sessionIds?: string[];
+  tokensBilled?: number;
 };
 
 // Reduce TTL to 6 hours instead of 24

apps/api/src/lib/extract/extraction-service.ts

@@ -1025,6 +1025,7 @@ export async function performExtraction(
         status: "completed",
         llmUsage,
         sources,
+        tokensBilled: tokensToBill,
         // costTracking,
       }).catch((error) => {
         logger.error(

apps/api/src/lib/extract/fire-0/extraction-service-f0.ts

@@ -866,6 +866,7 @@ import { getACUCTeam } from "../../../controllers/auth";
         status: "completed",
         llmUsage,
         sources,
+        tokensBilled: tokensToBill,
       }).catch((error) => {
         logger.error(
           `Failed to update extract ${extractId} status to completed: ${error}`,

apps/api/src/services/queue-worker.ts

@@ -1462,6 +1462,8 @@ async function processJob(job: Job & { id: string }, token: string) {
 
       const credits_billed = await billScrapeJob(job, doc, logger, costTracking);
 
+      doc.metadata.creditsUsed = credits_billed ?? undefined;
+
       logger.debug("Logging job to DB...");
       await logJob(
         {
@@ -1515,6 +1517,8 @@ async function processJob(job: Job & { id: string }, token: string) {
 
       const credits_billed = await billScrapeJob(job, doc, logger, costTracking);
 
+      doc.metadata.creditsUsed = credits_billed ?? undefined;
+
       await logJob({
         job_id: job.id,
         success: true,