yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-12-02 01:52:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Nick: got rid of job interval sleep and math.min

Browse Source
This commit is contained in:
Nicolas 2024-10-01 16:11:12 -03:00
parent 18f9cd09e1
commit c6717fecaa
2 changed files with 4 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/controllers/v1/concurrency-check.ts
View File

@ -12,7 +12,7 @@ export async function concurrencyCheckController(
req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>, req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>,
res: Response<ConcurrencyCheckResponse> res: Response<ConcurrencyCheckResponse>
) { ) {
const concurrencyLimiterKey = "concurrency-limiter:" + req.params.teamId; const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;
const now = Date.now(); const now = Date.now();
const activeJobsOfTeam = await redisConnection.zrangebyscore( const activeJobsOfTeam = await redisConnection.zrangebyscore(
concurrencyLimiterKey, concurrencyLimiterKey,

10
apps/api/src/services/queue-worker.ts
View File

@ -151,12 +151,8 @@ const workerFun = async (
await job.moveToFailed(new Error("Concurrency limit hit"), token, false); await job.moveToFailed(new Error("Concurrency limit hit"), token, false);
// Remove the job from the queue // Remove the job from the queue
await job.remove(); await job.remove();
// Increment the priority of the job exponentially by 5% // Increment the priority of the job exponentially by 5%, Note: max bull priority is 2 million
let newJobPriority = Math.round((job.opts.priority ?? 10) * 1.05); const newJobPriority = Math.min(Math.round((job.opts.priority ?? 10) * 1.05), 20000);
// Max priority is 200k, limit is 2 million
if(newJobPriority > 200000) {
newJobPriority = 200000;
}
// Add the job back to the queue with the new priority // Add the job back to the queue with the new priority
await queue.add(job.name, { await queue.add(job.name, {
...job.data, ...job.data,
@ -167,7 +163,7 @@ const workerFun = async (
priority: newJobPriority, // exponential backoff for stuck jobs priority: newJobPriority, // exponential backoff for stuck jobs
}); });
await sleep(gotJobInterval); // await sleep(gotJobInterval);
continue; continue;
} else { } else {
// If we are not throttled, add the job back to the queue with the new priority // If we are not throttled, add the job back to the queue with the new priority