yujunjun/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2025-06-27 00:41:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(index): store short titles and descriptions (#1677)

Browse Source
This commit is contained in:
Gergő Móricz 2025-06-17 19:09:07 +02:00 committed by GitHub
parent 9710bdffc0
commit 5ca8e2e98e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 23 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
apps/api/src/scraper/scrapeURL/engines/index/index.ts
View File

@ -60,6 +60,27 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
return document;
}
let title = document.metadata.title ?? document.metadata.ogTitle ?? null;
let description = document.metadata.description ?? document.metadata.ogDescription ?? document.metadata.dcDescription ?? null;
if (typeof title === "string") {
title = title.trim();
if (title.length > 60) {
title = title.slice(0, 57) + "...";
}
} else {
title = null;
}
if (typeof description === "string") {
description = description.trim();
if (description.length > 160) {
description = description.slice(0, 157) + "...";
}
} else {
description = null;
}
try {
await addIndexInsertJob({
id: indexId,
@ -82,6 +103,8 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
...a,
[`domain_splits_${i}_hash`]: x,
}), {})),
...(title ? { title } : {}),
...(description ? { description } : {}),
});
} catch (error) {
meta.logger.error("Failed to add document to index insert queue", {