mirror of
https://github.com/microsoft/graphrag.git
synced 2025-09-17 20:24:20 +00:00
379 lines
22 KiB
HTML
379 lines
22 KiB
HTML
|
|
|
|
|
|
|
|
<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Indexing Dataflow</title>
|
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
|
|
<link href="https://unpkg.com/prismjs@1.20.0/themes/prism-okaidia.css" rel="stylesheet">
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/Primer/19.1.1/tooltips.min.css" crossorigin="anonymous" referrerpolicy="no-referrer">
|
|
<style>
|
|
html {
|
|
padding: 0;
|
|
margin: 0;
|
|
}
|
|
|
|
body{
|
|
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
|
padding: 0;
|
|
margin: 0;
|
|
}
|
|
|
|
footer{
|
|
width: 100%;
|
|
height: 32px;
|
|
font-size: 12px;
|
|
display: flex;
|
|
flex-direction: row;
|
|
justify-content: center;
|
|
gap: 18px;
|
|
align-items: center;
|
|
color: #5d5d5d;
|
|
background: #e9eaeb;
|
|
border-top: 1px solid #c4c5c6;
|
|
}
|
|
|
|
#cookiesManager{
|
|
cursor: pointer;
|
|
color: #485fc7;
|
|
}
|
|
|
|
.page-content {
|
|
display: flex;
|
|
flex-direction: row;
|
|
margin: 0;
|
|
padding: 0;
|
|
overflow: scroll;
|
|
padding: 0;
|
|
margin: 0;
|
|
}
|
|
|
|
header {
|
|
background-color: lightgrey;
|
|
height: 2%;
|
|
padding: 10px;
|
|
}
|
|
|
|
nav {
|
|
padding: 1em;
|
|
min-width: 200px;
|
|
}
|
|
|
|
main {
|
|
flex: 1;
|
|
padding: 0 5em 0 5em;
|
|
}
|
|
|
|
.logotitle {
|
|
font-size: 1.5em;
|
|
font-weight: bold;
|
|
margin: 5px;
|
|
}
|
|
|
|
.number {
|
|
all: unset;
|
|
}
|
|
|
|
.tag.token {
|
|
all: unset;
|
|
}
|
|
|
|
main ul {
|
|
list-style-type: disc;
|
|
padding-left: 30px;
|
|
margin-top: 10px;
|
|
}
|
|
|
|
h1 {
|
|
font-size: 2rem;
|
|
margin-top: 10px;
|
|
}
|
|
|
|
h2 {
|
|
font-size: 1.5rem;
|
|
margin-top: 10px;
|
|
font-weight: 500;
|
|
}
|
|
|
|
h3 {
|
|
font-size: 1rem;
|
|
margin-top: 10px;
|
|
font-weight: 500;
|
|
}
|
|
p {
|
|
margin-top: 10px;
|
|
}
|
|
|
|
/* Accessibility styling */
|
|
|
|
a {
|
|
color: #485fc7;
|
|
text-decoration: underline;
|
|
}
|
|
|
|
.menu-list a {
|
|
text-decoration: none;
|
|
}
|
|
|
|
|
|
.token.comment, .token.prolog, .token.doctype, .token.cdata {
|
|
color: #8093a5;
|
|
}
|
|
|
|
.token.property, .token.tag, .token.constant, .token.symbol, .token.deleted {
|
|
color: #ff36ab;
|
|
}
|
|
</style>
|
|
<script type="module" async="">import mermaid from "https://unpkg.com/mermaid@10/dist/mermaid.esm.min.mjs";document.addEventListener('DOMContentLoaded', mermaid.initialize({"loadOnSave":true}));</script>
|
|
<script>function showTooltip(o,e){o.trigger.className.includes("tooltipped")||(o.trigger.children[0].className="tooltipped tooltipped-s",o.trigger.children[0].ariaLabel=e)}window.addEventListener("load",()=>{var o=new ClipboardJS(".code-copy");o.on("success",o=>showTooltip(o,"Copied!")),o.on("error",o=>showTooltip(o,"Failed..."))});</script>
|
|
<script async="" src="https://cdn.jsdelivr.net/npm/clipboard@2.0.11/dist/clipboard.min.js"></script>
|
|
|
|
|
|
<script src="https://wcpstatic.microsoft.com/mscc/lib/v2/wcp-consent.js" type="text/javascript"></script>
|
|
<script>
|
|
function onConsentChanged(categoryPreferences) {
|
|
console.log("onConsentChanged", categoryPreferences);
|
|
}
|
|
|
|
var siteConsent
|
|
|
|
function initialize(){
|
|
var currentYear = new Date().getFullYear()
|
|
document.getElementById("copyright").innerHTML = `©️ ${currentYear} Microsoft`;
|
|
window.WcpConsent && WcpConsent.init("en-US", "cookie-banner", function (err, _siteConsent) {
|
|
if (!err) {
|
|
siteConsent = _siteConsent; //siteConsent is used to get the current consent
|
|
} else {
|
|
console.log("Error initializing WcpConsent: "+ err);
|
|
}
|
|
}, onConsentChanged, WcpConsent.themes.light);
|
|
}
|
|
|
|
addEventListener("DOMContentLoaded", initialize)
|
|
addEventListener("DOMContentLoaded", checkCookieManager)
|
|
|
|
function checkCookieManager(){
|
|
if(siteConsent.isConsentRequired){
|
|
document.getElementById("cookiesManager").style.display = 'block';
|
|
document.getElementById("divider").style.display = 'block';
|
|
}
|
|
else{
|
|
document.getElementById("cookiesManager").style.display = 'none';
|
|
document.getElementById("divider").style.display = 'none';
|
|
}
|
|
}
|
|
|
|
function manageConsent() {
|
|
if(siteConsent.isConsentRequired){
|
|
siteConsent.manageConsent();
|
|
}
|
|
}
|
|
</script>
|
|
|
|
</head>
|
|
<body>
|
|
<header>
|
|
<div id="cookie-banner"></div>
|
|
<a href="/"><span class="logotitle">GraphRAG</span></a>
|
|
</header>
|
|
<div class="page-content">
|
|
<!-- Sidebar -->
|
|
<aside class="menu">
|
|
<ul class="menu-list">
|
|
<li>
|
|
|
|
<a href="/">Welcome</a>
|
|
|
|
</li>
|
|
|
|
<!-- Get Started Links -->
|
|
<li>
|
|
|
|
<a href="/posts/get_started/">Get Started</a>
|
|
|
|
|
|
<a href="/posts/developing/">Developing</a>
|
|
|
|
</li>
|
|
|
|
<!-- Indexing Links -->
|
|
<li>
|
|
|
|
<a href="/posts/index/overview/">Indexing</a>
|
|
|
|
<ul><li>
|
|
<a href="/posts/index/0-architecture/">Architecture</a>
|
|
</li><li>
|
|
<a href="/posts/index/1-default_dataflow/" class="is-active" aria-current="page">Dataflow</a>
|
|
</li><li>
|
|
<a href="/posts/index/2-cli/">CLI</a>
|
|
</li><li>
|
|
|
|
<a href="/posts/config/overview/">Configuration</a>
|
|
|
|
<ul>
|
|
<li>
|
|
<a href="/posts/config/env_vars">Using Env Vars</a>
|
|
</li>
|
|
<li>
|
|
<a href="/posts/config/json_yaml">Using JSON or YAML</a>
|
|
</li>
|
|
<li>
|
|
<a href="/posts/config/custom">Fully Custom</a>
|
|
</li>
|
|
<li>
|
|
<a href="/posts/config/template">Template</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
|
|
<li>
|
|
|
|
<a href="/posts/prompt_tuning/overview/">Prompt Tuning</a>
|
|
|
|
<ul>
|
|
<li>
|
|
|
|
<a href="/posts/prompt_tuning/auto_prompt_tuning/">Automatic Templating</a>
|
|
|
|
</li>
|
|
<li>
|
|
|
|
<a href="/posts/prompt_tuning/manual_prompt_tuning/">Manual Prompt Tuning</a>
|
|
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
|
|
|
|
<!-- Query Links -->
|
|
<li>
|
|
|
|
<a href="/posts/query/overview/">Query</a>
|
|
|
|
<ul><li>
|
|
<a href="/posts/query/1-local_search/">Local Search</a>
|
|
</li><li>
|
|
<a href="/posts/query/2-question_generation/">Question Generation</a>
|
|
</li><li>
|
|
<a href="/posts/query/0-global_search/">Global Search</a>
|
|
</li><li>
|
|
<a href="/posts/query/3-cli/">CLI</a>
|
|
</li><li>
|
|
|
|
<a href="/posts/query/notebooks/overview/">Notebooks</a>
|
|
|
|
<ul>
|
|
<li>
|
|
<a href="/posts/query/notebooks/global_search_nb">Global Search</a>
|
|
</li>
|
|
<li>
|
|
<a href="/posts/query/notebooks/local_search_nb">Local Search</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</aside>
|
|
|
|
<!-- Main Content -->
|
|
<main>
|
|
<h1>Indexing Dataflow</h1>
|
|
<h2>The GraphRAG Knowledge Model</h2>
|
|
<p>The knowledge model is a specification for data outputs that conform to our data-model definition. You can find these definitions in the python/graphrag/graphrag/model folder within the GraphRAG repository. The following entity types are provided. The fields here represent the fields that are text-embedded by default.</p>
|
|
<ul>
|
|
<li><code>Document</code> - An input document into the system. These either represent individual rows in a CSV or individual .txt file.</li>
|
|
<li><code>TextUnit</code> - A chunk of text to analyze. The size of these chunks, their overlap, and whether they adhere to any data boundaries may be configured below. A common use case is to set <code>CHUNK_BY_COLUMNS</code> to <code>id</code> so that there is a 1-to-many relationship between documents and TextUnits instead of a many-to-many.</li>
|
|
<li><code>Entity</code> - An entity extracted from a TextUnit. These represent people, places, events, or some other entity-model that you provide.</li>
|
|
<li><code>Relationship</code> - A relationship between two entities. These are generated from the covariates.</li>
|
|
<li><code>Covariate</code> - Extracted claim information, which contains statements about entities which may be time-bound.</li>
|
|
<li><code>Community Report</code> - Once entities are generated, we perform hierarchical community detection on them and generate reports for each community in this hierarchy.</li>
|
|
<li><code>Node</code> - This table contains layout information for rendered graph-views of the Entities and Documents which have been embedded and clustered.</li>
|
|
</ul>
|
|
<h2>The Default Configuration Workflow</h2>
|
|
<p>Let's take a look at how the default-configuration workflow transforms text documents into the <em>GraphRAG Knowledge Model</em>. This page gives a general overview of the major steps in this process. To fully configure this workflow, check out the <a href="/posts/config/overview/">configuration</a> documentation.</p>
|
|
<pre class="mermaid">--- title: Dataflow Overview --- flowchart TB subgraph phase1[Phase 1: Compose TextUnits] documents[Documents] --> chunk[Chunk] chunk --> embed[Embed] --> textUnits[Text Units] end subgraph phase2[Phase 2: Graph Extraction] textUnits --> graph_extract[Entity & Relationship Extraction] graph_extract --> graph_summarize[Entity & Relationship Summarization] graph_summarize --> entity_resolve[Entity Resolution] entity_resolve --> claim_extraction[Claim Extraction] claim_extraction --> graph_outputs[Graph Tables] end subgraph phase3[Phase 3: Graph Augmentation] graph_outputs --> community_detect[Community Detection] community_detect --> graph_embed[Graph Embedding] graph_embed --> augmented_graph[Augmented Graph Tables] end subgraph phase4[Phase 4: Community Summarization] augmented_graph --> summarized_communities[Community Summarization] summarized_communities --> embed_communities[Community Embedding] embed_communities --> community_outputs[Community Tables] end subgraph phase5[Phase 5: Document Processing] documents --> link_to_text_units[Link to TextUnits] textUnits --> link_to_text_units link_to_text_units --> embed_documents[Document Embedding] embed_documents --> document_graph[Document Graph Creation] document_graph --> document_outputs[Document Tables] end subgraph phase6[Phase 6: Network Visualization] document_outputs --> umap_docs[Umap Documents] augmented_graph --> umap_entities[Umap Entities] umap_docs --> combine_nodes[Nodes Table] umap_entities --> combine_nodes end </pre>
|
|
<h2>Phase 1: Compose TextUnits</h2>
|
|
<p>The first phase of the default-configuration workflow is to transform input documents into <em>TextUnits</em>. A <em>TextUnit</em> is a chunk of text that is used for our graph extraction techniques. They are also used as source-references by extracted knowledge items in order to empower breadcrumbs and provenance by concepts back to their original source tex.</p>
|
|
<p>The chunk size (counted in tokens), is user-configurable. By default this is set to 300 tokens, although we've had positive experience with 1200-token chunks using a single "glean" step. (A "glean" step is a follow-on extraction). Larger chunks result in lower-fidelity output and less meaningful reference texts; however, using larger chunks can result in much faster processing time.</p>
|
|
<p>The group-by configuration is also user-configurable. By default, we align our chunks to document boundaries, meaning that there is a strict 1-to-many relationship between Documents and TextUnits. In rare cases, this can be turned into a many-to-many relationship. This is useful when the documents are very short and we need several of them to compose a meaningful analysis unit (e.g. Tweets or a chat log)</p>
|
|
<p>Each of these text-units are text-embedded and passed into the next phase of the pipeline.</p>
|
|
<pre class="mermaid">--- title: Documents into Text Chunks --- flowchart LR doc1[Document 1] --> tu1[TextUnit 1] doc1 --> tu2[TextUnit 2] doc2[Document 2] --> tu3[TextUnit 3] doc2 --> tu4[TextUnit 4] </pre>
|
|
<h2>Phase 2: Graph Extraction</h2>
|
|
<p>In this phase, we analyze each text unit and extract our graph primitives: <em>Entities</em>, <em>Relationships</em>, and <em>Claims</em>.
|
|
Entities and Relationships are extracted at once in our <em>entity_extract</em> verb, and claims are extracted in our <em>claim_extract</em> verb. Results are then combined and passed into following phases of the pipeline.</p>
|
|
<pre class="mermaid">--- title: Graph Extraction --- flowchart LR tu[TextUnit] --> ge[Graph Extraction] --> gs[Graph Summarization] --> er[Entity Resolution] tu --> ce[Claim Extraction] </pre>
|
|
<h3>Entity & Relationship Extraction</h3>
|
|
<p>In this first step of graph extraction, we process each text-unit in order to extract entities and relationships out of the raw text using the LLM. The output of this step is a subgraph-per-TextUnit containing a list of <strong>entities</strong> with a <em>name</em>, <em>type</em>, and <em>description</em>, and a list of <strong>relationships</strong> with a <em>source</em>, <em>target</em>, and <em>description</em>.</p>
|
|
<p>These subgraphs are merged together - any entities with the same <em>name</em> and <em>type</em> are merged by creating an array of their descriptions. Similarly, any relationships with the same <em>source</em> and <em>target</em> are merged by creating an array of their descriptions.</p>
|
|
<h3>Entity & Relationship Summarization</h3>
|
|
<p>Now that we have a graph of entities and relationships, each with a list of descriptions, we can summarize these lists into a single description per entity and relationship. This is done by asking the LLM for a short summary that captures all of the distinct information from each description. This allows all of our entities and relationships to have a single concise description.</p>
|
|
<h3>Entity Resolution (Not Enabled by Default)</h3>
|
|
<p>The final step of graph extraction is to resolve any entities that represent the same real-world entity but but have different names. Since this is done via LLM, and we don't want to lose information, we want to take a conservative, non-destructive approach to this.</p>
|
|
<p>Our current implementation of Entity Resolution, however, is destructive. It will provide the LLM with a series of entities and ask it to determine which ones should be merged. Those entities are then merged together into a single entity and their relationships are updated.</p>
|
|
<p>We are currently exploring other entity resolution techniques. In the near future, entity resolution will be executed by creating an edge between entity variants indicating that the entities have been resolved by the indexing engine. This will allow for end-users to undo indexing-side resolutions, and add their own non-destructive resolutions using a similar process.</p>
|
|
<h3>Claim Extraction & Emission</h3>
|
|
<p>Finally, as an independent workflow, we extract claims from the source TextUnits. These claims represent positive factual statements with an evaluated status and time-bounds. These are emitted as a primary artifact called <strong>Covariates</strong>.</p>
|
|
<h2>Phase 3: Graph Augmentation</h2>
|
|
<p>Now that we have a usable graph of entities and relationships, we want to understand their community structure and augment the graph with additional information. This is done in two steps: <em>Community Detection</em> and <em>Graph Embedding</em>. These give us explicit (communities) and implicit (embeddings) ways of understanding the topological structure of our graph.</p>
|
|
<pre class="mermaid">--- title: Graph Augmentation --- flowchart LR cd[Leiden Hierarchical Community Detection] --> ge[Node2Vec Graph Embedding] --> ag[Graph Table Emission] </pre>
|
|
<h3>Community Detection</h3>
|
|
<p>In this step, we generate a hierarchy of entity communities using the Hierarchical Leiden Algorithm. This method will apply a recursive community-clustering to our graph until we reach a community-size threshold. This will allow us to understand the community structure of our graph and provide a way to navigate and summarize the graph at different levels of granularity.</p>
|
|
<h3>Graph Embedding</h3>
|
|
<p>In this step, we generate a vector representation of our graph using the Node2Vec algorithm. This will allow us to understand the implicit structure of our graph and provide an additional vector-space in which to search for related concepts during our query phase.</p>
|
|
<h3>Graph Tables Emission</h3>
|
|
<p>Once our graph augmentation steps are complete, the final <strong>Entities</strong> and <strong>Relationships</strong> tables are emitted after their text fields are text-embedded.</p>
|
|
<h2>Phase 4: Community Summarization</h2>
|
|
<pre class="mermaid">--- title: Community Summarization --- flowchart LR sc[Generate Community Reports] --> ss[Summarize Community Reports] --> ce[Community Embedding] --> co[Community Tables Emission] </pre>
|
|
<p>At this point, we have a functional graph of entities and relationships, a hierarchy of communities for the entities, as well as node2vec embeddings.</p>
|
|
<p>Now we want to build on the communities data and generate reports for each community. This gives us a high-level understanding of the graph at several points of graph granularity. For example, if community A is the top-level community, we'll get a report about the entire graph. If the community is lower-level, we'll get a report about a local cluster.</p>
|
|
<h3>Generate Community Reports</h3>
|
|
<p>In this step, we generate a summary of each community using the LLM. This will allow us to understand the distinct information contained within each community and provide a scoped understanding of the graph, from either a high-level or a low-level perspective. These reports contain an executive overview and reference the key entities, relationships, and claims within the community sub-structure.</p>
|
|
<h3>Summarize Community Reports</h3>
|
|
<p>In this step, each <em>community report</em> is then summarized via the LLM for shorthand use.</p>
|
|
<h3>Community Embedding</h3>
|
|
<p>In this step, we generate a vector representation of our communities by generating text embeddings of the community report, the community report summary, and the title of the community report.</p>
|
|
<h3>Community Tables Emission</h3>
|
|
<p>At this point, some bookkeeping work is performed and we emit the <strong>Communities</strong> and <strong>CommunityReports</strong> tables.</p>
|
|
<h2>Phase 5: Document Processing</h2>
|
|
<p>In this phase of the workflow, we create the <em>Documents</em> table for the knowledge model.</p>
|
|
<pre class="mermaid">--- title: Document Processing --- flowchart LR aug[Augment] --> dp[Link to TextUnits] --> de[Avg. Embedding] --> dg[Document Table Emission] </pre>
|
|
<h3>Augment with Columns (CSV Only)</h3>
|
|
<p>If the workflow is operating on CSV data, you may configure your workflow to add additional fields to Documents output. These fields should exist on the incoming CSV tables. Details about configuring this can be found in the <a href="/posts/config/overview/">configuration documentation</a>.</p>
|
|
<h3>Link to TextUnits</h3>
|
|
<p>In this step, we link each document to the text-units that were created in the first phase. This allows us to understand which documents are related to which text-units and vice-versa.</p>
|
|
<h3>Document Embedding</h3>
|
|
<p>In this step, we generate a vector representation of our documents using an average embedding of document slices. We re-chunk documents without overlapping chunks, and then generate an embedding for each chunk. We create an average of these chunks weighted by token-count and use this as the document embedding. This will allow us to understand the implicit relationship between documents, and will help us generate a network representation of our documents.</p>
|
|
<h3>Documents Table Emission</h3>
|
|
<p>At this point, we can emit the <strong>Documents</strong> table into the knowledge Model.</p>
|
|
<h2>Phase 6: Network Visualization</h2>
|
|
<p>In this phase of the workflow, we perform some steps to support network visualization of our high-dimensional vector spaces within our existing graphs. At this point there are two logical graphs at play: the <em>Entity-Relationship</em> graph and the <em>Document</em> graph.</p>
|
|
<pre class="mermaid">--- title: Network Visualization Workflows --- flowchart LR nv[Umap Documents] --> ne[Umap Entities] --> ng[Nodes Table Emission] </pre>
|
|
<p>For each of the logical graphs, we perform a UMAP dimensionality reduction to generate a 2D representation of the graph. This will allow us to visualize the graph in a 2D space and understand the relationships between the nodes in the graph. The UMAP embeddings are then emitted as a table of <em>Nodes</em>. The rows of this table include a discriminator indicating whether the node is a document or an entity, and the UMAP coordinates.</p>
|
|
|
|
</main>
|
|
</div>
|
|
<footer>
|
|
<a href="https://go.microsoft.com/fwlink/?LinkId=521839">Privacy</a>
|
|
|
|
|
<a href="https://go.microsoft.com/fwlink/?LinkId=2259814">Consumer Health Privacy</a>
|
|
|
|
|
<span id="cookiesManager" onClick="manageConsent();">Cookies</span>
|
|
<span id="divider">|</span>
|
|
<a href="https://go.microsoft.com/fwlink/?LinkID=206977">Terms of Use</a>
|
|
|
|
|
<a href="https://www.microsoft.com/trademarks">Trademarks</a>
|
|
|
|
|
<a href="https://www.microsoft.com" id="copyright"></a>
|
|
|
|
|
<a href="https://github.com/microsoft/graphrag">GitHub</a>
|
|
</footer>
|
|
</body>
|
|
</html> |