yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-06-27 00:41:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Nick: improved map ranking algorithm"

Browse Source 
This reverts commit 7acd8d2edb6abc45a63fe1060377d2acb398ec36.
This commit is contained in:
Nicolas 2024-10-21 16:11:32 -03:00
parent edac6850c6
commit d2344aa14b
3 changed files with 8 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/controllers/v1/map.ts
View File

@ -2,7 +2,6 @@ import { Response } from "express";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
import { import {
legacyCrawlerOptions, legacyCrawlerOptions,
LinkInfo,
mapRequestSchema, mapRequestSchema,
RequestWithAuth, RequestWithAuth,
} from "./types"; } from "./types";
@ -110,10 +109,6 @@ export async function mapController(
mapResults = mapResults.slice(0, minumumCutoff); mapResults = mapResults.slice(0, minumumCutoff);
} }
let linkInfos: LinkInfo[] = [];
if (mapResults.length > 0) { if (mapResults.length > 0) {
if (req.body.search) { if (req.body.search) {
// Ensure all map results are first, maintaining their order // Ensure all map results are first, maintaining their order
@ -122,12 +117,6 @@ export async function mapController(
...mapResults.slice(1).map((x) => x.url), ...mapResults.slice(1).map((x) => x.url),
...links, ...links,
]; ];
linkInfos = [
mapResults[0],
...mapResults.slice(1),
...links.map((x) => ({ url: x })),
]
} else { } else {
mapResults.map((x) => { mapResults.map((x) => {
links.push(x.url); links.push(x.url);
@ -139,7 +128,7 @@ export async function mapController(
if (req.body.search) { if (req.body.search) {
const searchQuery = req.body.search.toLowerCase(); const searchQuery = req.body.search.toLowerCase();
links = performCosineSimilarity(linkInfos, searchQuery); links = performCosineSimilarity(links, searchQuery);
} }
links = links links = links

8
apps/api/src/controllers/v1/types.ts
View File

@ -478,11 +478,3 @@ export function legacyDocumentConverter(doc: any): Document {
}, },
}; };
} }
export interface LinkInfo {
url: string;
title?: string;
description?: string;
}

15
apps/api/src/lib/map-cosine.ts
View File

@ -1,7 +1,6 @@
import { Logger } from "./logger"; import { Logger } from "./logger";
import { LinkInfo } from "../controllers/v1/types";
export function performCosineSimilarity(links: LinkInfo[], searchQuery: string) { export function performCosineSimilarity(links: string[], searchQuery: string) {
try { try {
// Function to calculate cosine similarity // Function to calculate cosine similarity
const cosineSimilarity = (vec1: number[], vec2: number[]): number => { const cosineSimilarity = (vec1: number[], vec2: number[]): number => {
@ -28,20 +27,20 @@ export function performCosineSimilarity(links: LinkInfo[], searchQuery: string)
// Calculate similarity scores // Calculate similarity scores
const similarityScores = links.map((link) => { const similarityScores = links.map((link) => {
const linkText = `${link.url} ${link.title || ''} ${link.description || ''}`.trim(); const linkVector = textToVector(link);
const linkVector = textToVector(linkText);
const searchVector = textToVector(searchQuery); const searchVector = textToVector(searchQuery);
return cosineSimilarity(linkVector, searchVector); return cosineSimilarity(linkVector, searchVector);
}); });
// Sort links based on similarity scores // Sort links based on similarity scores and print scores
const sortedLinks = links const a = links
.map((link, index) => ({ link, score: similarityScores[index] })) .map((link, index) => ({ link, score: similarityScores[index] }))
.sort((a, b) => b.score - a.score); .sort((a, b) => b.score - a.score);
return sortedLinks.map((item) => item.link.url); links = a.map((item) => item.link);
return links;
} catch (error) { } catch (error) {
Logger.error(`Error performing cosine similarity: ${error}`); Logger.error(`Error performing cosine similarity: ${error}`);
return links.map(link => link.url); return links;
} }
} }