yujunjun/LightRAG
1
0
Fork 0
mirror of https://github.com/HKUDS/LightRAG.git synced 2025-06-26 22:00:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve the pipeline status message for document deletetion

Browse Source
This commit is contained in:
yangdx 2025-06-25 15:46:58 +08:00
parent 2aaa6d5f7d
commit 495d6c8cce
3 changed files with 87 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lightrag/api/routers/document_routes.py
View File

@ -803,9 +803,7 @@ async def background_delete_document(rag: LightRAG, doc_id: str):
}
)
# Use slice assignment to clear the list in place
pipeline_status["history_messages"][:] = [
f"Starting deletion for doc_id: {doc_id}"
]
pipeline_status["history_messages"][:] = ["Starting document deletion process"]
try:
result = await rag.adelete_by_doc_id(doc_id)
@ -823,7 +821,7 @@ async def background_delete_document(rag: LightRAG, doc_id: str):
finally:
async with pipeline_status_lock:
pipeline_status["busy"] = False
completion_msg = f"Document deletion process for {doc_id} completed."
completion_msg = "Document deletion process completed."
pipeline_status["latest_message"] = completion_msg
if "history_messages" in pipeline_status:
pipeline_status["history_messages"].append(completion_msg)

144
lightrag/lightrag.py
View File

@ -1683,15 +1683,7 @@ class LightRAG:
This method orchestrates a comprehensive deletion process for a given document ID.
It ensures that not only the document itself but also all its derived and associated
data across different storage layers are removed. This includes:
1. **Document and Status**: Deletes the document from `full_docs` and its status from `doc_status`.
2. **Chunks**: Removes all associated text chunks from `chunks_vdb`.
3. **Graph Data**:
- Deletes related entities from `entities_vdb`.
- Deletes related relationships from `relationships_vdb`.
- Removes corresponding nodes and edges from the `chunk_entity_relation_graph`.
4. **Graph Reconstruction**: If entities or relationships are partially affected, it triggers
a reconstruction of their data from the remaining chunks to ensure consistency.
data across different storage layers are removed. If entities or relationships are partially affected, it triggers.
Args:
doc_id (str): The unique identifier of the document to be deleted.
@ -1706,9 +1698,17 @@ class LightRAG:
deletion_operations_started = False
original_exception = None
try:
logger.info(f"Starting deletion process for document {doc_id}")
# Get pipeline status shared data and lock for status updates
pipeline_status = await get_namespace_data("pipeline_status")
pipeline_status_lock = get_pipeline_status_lock()
async with pipeline_status_lock:
log_message = f"Starting deletion process for document {doc_id}"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
try:
# 1. Get the document status and related data
doc_status_data = await self.doc_status.get_by_id(doc_id)
if not doc_status_data:
@ -1720,8 +1720,6 @@ class LightRAG:
status_code=404,
)
logger.info(f"Starting optimized deletion for document {doc_id}")
# 2. Get all chunks related to this document
try:
all_chunks = await self.text_chunks.get_all()
@ -1731,9 +1729,14 @@ class LightRAG:
if isinstance(chunk_data, dict)
and chunk_data.get("full_doc_id") == doc_id
}
logger.info(
f"Retrieved {len(all_chunks)} total chunks, {len(related_chunks)} related to document {doc_id}"
)
# Update pipeline status after getting chunks count
async with pipeline_status_lock:
log_message = f"Retrieved {len(related_chunks)} of {len(all_chunks)} related chunks"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
except Exception as e:
logger.error(f"Failed to retrieve chunks for document {doc_id}: {e}")
raise Exception(f"Failed to retrieve document chunks: {e}") from e
@ -1753,16 +1756,22 @@ class LightRAG:
)
raise Exception(f"Failed to delete document entry: {e}") from e
async with pipeline_status_lock:
log_message = (
f"Document {doc_id} is deleted without associated chunks."
)
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
return DeletionResult(
status="success",
doc_id=doc_id,
message=f"Document {doc_id} found but had no associated chunks. Document entry deleted.",
message=log_message,
status_code=200,
)
chunk_ids = set(related_chunks.keys())
logger.info(f"Found {len(chunk_ids)} chunks to delete")
# Mark that deletion operations have started
deletion_operations_started = True
@ -1777,22 +1786,35 @@ class LightRAG:
async with graph_db_lock:
try:
# Get all affected nodes and edges in batch
logger.info(
f"Analyzing affected entities and relationships for {len(chunk_ids)} chunks"
)
# logger.info(
# f"Analyzing affected entities and relationships for {len(chunk_ids)} chunks"
# )
affected_nodes = (
await self.chunk_entity_relation_graph.get_nodes_by_chunk_ids(
list(chunk_ids)
)
)
# Update pipeline status after getting affected_nodes
async with pipeline_status_lock:
log_message = f"Found {len(affected_nodes)} affected entities"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
affected_edges = (
await self.chunk_entity_relation_graph.get_edges_by_chunk_ids(
list(chunk_ids)
)
)
logger.info(
f"Found {len(affected_nodes)} affected nodes and {len(affected_edges)} affected edges"
)
# Update pipeline status after getting affected_edges
async with pipeline_status_lock:
log_message = f"Found {len(affected_edges)} affected relations"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
except Exception as e:
logger.error(f"Failed to analyze affected graph elements: {e}")
raise Exception(f"Failed to analyze graph dependencies: {e}") from e
@ -1831,12 +1853,6 @@ class LightRAG:
elif remaining_sources != sources:
relationships_to_rebuild[edge_tuple] = remaining_sources
logger.info(
f"Analysis complete: {len(entities_to_delete)} entities to delete, "
f"{len(entities_to_rebuild)} entities to rebuild, "
f"{len(relationships_to_delete)} relationships to delete, "
f"{len(relationships_to_rebuild)} relationships to rebuild"
)
except Exception as e:
logger.error(f"Failed to process graph analysis results: {e}")
raise Exception(f"Failed to process graph dependencies: {e}") from e
@ -1844,12 +1860,15 @@ class LightRAG:
# 5. Delete chunks from storage
if chunk_ids:
try:
logger.info(f"Deleting {len(chunk_ids)} chunks from storage")
await self.chunks_vdb.delete(chunk_ids)
await self.text_chunks.delete(chunk_ids)
logger.info(
f"Successfully deleted {len(chunk_ids)} chunks from storage"
)
async with pipeline_status_lock:
log_message = f"Successfully deleted {len(chunk_ids)} chunks from storage"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
except Exception as e:
logger.error(f"Failed to delete chunks: {e}")
raise Exception(f"Failed to delete document chunks: {e}") from e
@ -1857,7 +1876,6 @@ class LightRAG:
# 6. Delete entities that have no remaining sources
if entities_to_delete:
try:
logger.info(f"Deleting {len(entities_to_delete)} entities")
# Delete from vector database
entity_vdb_ids = [
compute_mdhash_id(entity, prefix="ent-")
@ -1869,9 +1887,13 @@ class LightRAG:
await self.chunk_entity_relation_graph.remove_nodes(
list(entities_to_delete)
)
logger.info(
f"Successfully deleted {len(entities_to_delete)} entities"
)
async with pipeline_status_lock:
log_message = f"Successfully deleted {len(entities_to_delete)} entities"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
except Exception as e:
logger.error(f"Failed to delete entities: {e}")
raise Exception(f"Failed to delete entities: {e}") from e
@ -1879,9 +1901,6 @@ class LightRAG:
# 7. Delete relationships that have no remaining sources
if relationships_to_delete:
try:
logger.info(
f"Deleting {len(relationships_to_delete)} relationships"
)
# Delete from vector database
rel_ids_to_delete = []
for src, tgt in relationships_to_delete:
@ -1897,9 +1916,13 @@ class LightRAG:
await self.chunk_entity_relation_graph.remove_edges(
list(relationships_to_delete)
)
logger.info(
f"Successfully deleted {len(relationships_to_delete)} relationships"
)
async with pipeline_status_lock:
log_message = f"Successfully deleted {len(relationships_to_delete)} relations"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
except Exception as e:
logger.error(f"Failed to delete relationships: {e}")
raise Exception(f"Failed to delete relationships: {e}") from e
@ -1907,9 +1930,6 @@ class LightRAG:
# 8. Rebuild entities and relationships from remaining chunks
if entities_to_rebuild or relationships_to_rebuild:
try:
logger.info(
f"Rebuilding {len(entities_to_rebuild)} entities and {len(relationships_to_rebuild)} relationships"
)
await _rebuild_knowledge_from_chunks(
entities_to_rebuild=entities_to_rebuild,
relationships_to_rebuild=relationships_to_rebuild,
@ -1920,9 +1940,13 @@ class LightRAG:
llm_response_cache=self.llm_response_cache,
global_config=asdict(self),
)
logger.info(
f"Successfully rebuilt {len(entities_to_rebuild)} entities and {len(relationships_to_rebuild)} relationships"
)
async with pipeline_status_lock:
log_message = f"Successfully rebuilt {len(entities_to_rebuild)} entities and {len(relationships_to_rebuild)} relations"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
except Exception as e:
logger.error(f"Failed to rebuild knowledge from chunks: {e}")
raise Exception(
@ -1931,20 +1955,22 @@ class LightRAG:
# 9. Delete original document and status
try:
logger.info(f"Deleting original document {doc_id} and its status")
await self.full_docs.delete([doc_id])
await self.doc_status.delete([doc_id])
logger.info(f"Successfully deleted document {doc_id} and its status")
except Exception as e:
logger.error(f"Failed to delete document and status: {e}")
raise Exception(f"Failed to delete document and status: {e}") from e
success_message = f"Successfully deleted document {doc_id}"
logger.info(success_message)
async with pipeline_status_lock:
log_message = f"Successfully deleted document {doc_id}"
logger.info(log_message)
pipeline_status["latest_message"] = log_message
pipeline_status["history_messages"].append(log_message)
return DeletionResult(
status="success",
doc_id=doc_id,
message=success_message,
message=log_message,
status_code=200,
)
@ -1964,13 +1990,7 @@ class LightRAG:
# ALWAYS ensure persistence if any deletion operations were started
if deletion_operations_started:
try:
logger.info(
f"Ensuring data persistence for document {doc_id} deletion"
)
await self._insert_done()
logger.info(
f"Data persistence completed successfully for document {doc_id} deletion"
)
except Exception as persistence_error:
persistence_error_msg = f"Failed to persist data after deletion attempt for {doc_id}: {persistence_error}"
logger.error(persistence_error_msg)

6
lightrag/operate.py
View File

@ -270,7 +270,7 @@ async def _rebuild_knowledge_from_chunks(
for chunk_ids in relationships_to_rebuild.values():
all_referenced_chunk_ids.update(chunk_ids)
logger.info(
logger.debug(
f"Rebuilding knowledge from {len(all_referenced_chunk_ids)} cached chunk extractions"
)
@ -339,7 +339,7 @@ async def _rebuild_knowledge_from_chunks(
except Exception as e:
logger.error(f"Failed to rebuild relationship {src}-{tgt}: {e}")
logger.info("Completed rebuilding knowledge from cached extractions")
logger.debug("Completed rebuilding knowledge from cached extractions")
async def _get_cached_extraction_results(
@ -368,7 +368,7 @@ async def _get_cached_extraction_results(
extraction_result = cache_entry["return"]
cached_results[chunk_id] = extraction_result
logger.info(
logger.debug(
f"Found {len(cached_results)} cached extraction results for {len(chunk_ids)} chunk IDs"
)
return cached_results