diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts index e45f49e96..febe11a57 100644 --- a/apps/api/src/controllers/v1/crawl-status.ts +++ b/apps/api/src/controllers/v1/crawl-status.ts @@ -22,7 +22,7 @@ import { configDotenv } from "dotenv"; import type { Job, JobState, Queue } from "bullmq"; import { logger } from "../../lib/logger"; import { supabase_rr_service, supabase_service } from "../../services/supabase"; -import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit"; +import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit"; import { getJobFromGCS } from "../../lib/gcs-jobs"; configDotenv(); @@ -163,6 +163,7 @@ export async function crawlStatusController( ); const throttledJobsSet = new Set(await getConcurrencyLimitedJobs(req.auth.team_id)); + const activeJobsSet = new Set(await getCrawlConcurrencyLimitActiveJobs(req.params.jobId)); const validJobStatuses: [string, JobState | "unknown"][] = []; const validJobIDs: string[] = []; @@ -171,6 +172,9 @@ export async function crawlStatusController( if (throttledJobsSet.has(id)) { validJobStatuses.push([id, "prioritized"]); validJobIDs.push(id); + } else if (status === "unknown" && activeJobsSet.has(id)) { + validJobStatuses.push([id, "active"]); + validJobIDs.push(id); } else if ( status !== "failed" && status !== "unknown" diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts index 400323b9a..8eb7813ea 100644 --- a/apps/api/src/lib/concurrency-limit.ts +++ b/apps/api/src/lib/concurrency-limit.ts @@ -216,8 +216,7 @@ async function getNextConcurrentJob(teamId: string): Promise<{ } for (const ignoredJob of ignoredJobs) { - const timeout = ignoredJob.timeout - Date.now(); - await pushConcurrencyLimitedJob(teamId, ignoredJob.job, timeout); + await pushConcurrencyLimitedJob(teamId, ignoredJob.job, ignoredJob.timeout); } return finalJob;