yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-06-27 00:41:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(extract-status): be able to get extract status even after TTL lapses (#1599)

Browse Source
This commit is contained in:
Gergő Móricz 2025-05-23 22:33:09 +02:00 committed by GitHub
parent 8389a1a78d
commit a3145ccacc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 31 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/controllers/v1/crawl-status.ts
View File

@ -33,11 +33,12 @@ export type PseudoJob<T> = {
timestamp: number,
data: {
scrapeOptions: any,
teamId?: string,
},
failedReason?: string,
}
export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null }
export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string}
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([

43
apps/api/src/controllers/v1/extract-status.ts
View File

@ -18,10 +18,11 @@ export async function getExtractJob(id: string): Promise<PseudoJob<ExtractResult
const job: PseudoJob<any> = {
id,
getState: bullJob ? bullJob.getState : (() => dbJob!.success ? "completed" : "failed"),
getState: bullJob ? bullJob.getState.bind(bullJob) : (() => dbJob!.success ? "completed" : "failed"),
returnvalue: data,
data: {
scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,
teamId: bullJob ? bullJob.data.teamId : dbJob!.team_id,
},
timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),
failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
@ -36,7 +37,9 @@ export async function extractStatusController(
) {
const extract = await getExtract(req.params.jobId);
if (!extract) {
let status = extract?.status;
if (extract && extract.team_id !== req.auth.team_id) {
return res.status(404).json({
success: false,
error: "Extract job not found",
@ -45,34 +48,46 @@ export async function extractStatusController(
let data: ExtractResult | [] = [];
if (extract.status === "completed") {
if (!extract || extract.status === "completed") {
const jobData = await getExtractJob(req.params.jobId);
if (!jobData) {
if ((!jobData && !extract) || (jobData && jobData.data.teamId !== req.auth.team_id)) {
return res.status(404).json({
success: false,
error: "Job not found",
error: "Extract job not found",
});
}
if (!jobData.returnvalue) {
if (jobData) {
const jobStatus = await jobData.getState();
if (jobStatus === "completed") {
status = "completed";
} else if (jobStatus === "failed") {
status = "failed";
} else {
status = "processing";
}
}
if (!jobData?.returnvalue) {
// if we got in the split-second where the redis is updated but the bull isn't
// just pretend it's still processing - MG
extract.status = "processing";
status = "processing";
} else {
data = jobData.returnvalue ?? [];
}
}
return res.status(200).json({
success: extract.status === "failed" ? false : true,
success: status === "failed" ? false : true,
data,
status: extract.status,
status,
error: extract?.error ?? undefined,
expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),
steps: extract.showSteps ? extract.steps : undefined,
llmUsage: extract.showLLMUsage ? extract.llmUsage : undefined,
sources: extract.showSources ? extract.sources : undefined,
costTracking: extract.showCostTracking ? extract.costTracking : undefined,
sessionIds: extract.sessionIds ? extract.sessionIds : undefined,
steps: extract?.showSteps ? extract.steps : undefined,
llmUsage: extract?.showLLMUsage ? extract.llmUsage : undefined,
sources: extract?.showSources ? extract.sources : undefined,
costTracking: extract?.showCostTracking ? extract.costTracking : undefined,
sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,
});
}