mirror of
https://github.com/microsoft/graphrag.git
synced 2025-10-31 09:50:13 +00:00
390 lines
13 KiB
HTML
390 lines
13 KiB
HTML
|
|
|
|
|
|
|
|
<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Developing GraphRAG</title>
|
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
|
|
<link href="https://unpkg.com/prismjs@1.20.0/themes/prism-okaidia.css" rel="stylesheet">
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/Primer/19.1.1/tooltips.min.css" crossorigin="anonymous" referrerpolicy="no-referrer">
|
|
<style>
|
|
html {
|
|
padding: 0;
|
|
margin: 0;
|
|
}
|
|
|
|
body{
|
|
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
|
padding: 0;
|
|
margin: 0;
|
|
}
|
|
|
|
footer{
|
|
width: 100%;
|
|
height: 32px;
|
|
font-size: 12px;
|
|
display: flex;
|
|
flex-direction: row;
|
|
justify-content: center;
|
|
gap: 18px;
|
|
align-items: center;
|
|
color: #5d5d5d;
|
|
background: #e9eaeb;
|
|
border-top: 1px solid #c4c5c6;
|
|
}
|
|
|
|
#cookiesManager{
|
|
cursor: pointer;
|
|
color: #485fc7;
|
|
}
|
|
|
|
.page-content {
|
|
display: flex;
|
|
flex-direction: row;
|
|
margin: 0;
|
|
padding: 0;
|
|
overflow: scroll;
|
|
padding: 0;
|
|
margin: 0;
|
|
}
|
|
|
|
header {
|
|
background-color: lightgrey;
|
|
height: 2%;
|
|
padding: 10px;
|
|
}
|
|
|
|
nav {
|
|
padding: 1em;
|
|
min-width: 200px;
|
|
}
|
|
|
|
main {
|
|
flex: 1;
|
|
padding: 0 5em 0 5em;
|
|
}
|
|
|
|
.logotitle {
|
|
font-size: 1.5em;
|
|
font-weight: bold;
|
|
margin: 5px;
|
|
}
|
|
|
|
.number {
|
|
all: unset;
|
|
}
|
|
|
|
.tag.token {
|
|
all: unset;
|
|
}
|
|
|
|
main ul {
|
|
list-style-type: disc;
|
|
padding-left: 30px;
|
|
margin-top: 10px;
|
|
}
|
|
|
|
h1 {
|
|
font-size: 2rem;
|
|
margin-top: 10px;
|
|
}
|
|
|
|
h2 {
|
|
font-size: 1.5rem;
|
|
margin-top: 10px;
|
|
font-weight: 500;
|
|
}
|
|
|
|
h3 {
|
|
font-size: 1rem;
|
|
margin-top: 10px;
|
|
font-weight: 500;
|
|
}
|
|
p {
|
|
margin-top: 10px;
|
|
}
|
|
|
|
/* Accessibility styling */
|
|
|
|
a {
|
|
color: #485fc7;
|
|
text-decoration: underline;
|
|
}
|
|
|
|
.menu-list a {
|
|
text-decoration: none;
|
|
}
|
|
|
|
|
|
.token.comment, .token.prolog, .token.doctype, .token.cdata {
|
|
color: #8093a5;
|
|
}
|
|
|
|
.token.property, .token.tag, .token.constant, .token.symbol, .token.deleted {
|
|
color: #ff36ab;
|
|
}
|
|
</style>
|
|
<script type="module" async="">import mermaid from "https://unpkg.com/mermaid@10/dist/mermaid.esm.min.mjs";document.addEventListener('DOMContentLoaded', mermaid.initialize({"loadOnSave":true}));</script>
|
|
<script>function showTooltip(o,e){o.trigger.className.includes("tooltipped")||(o.trigger.children[0].className="tooltipped tooltipped-s",o.trigger.children[0].ariaLabel=e)}window.addEventListener("load",()=>{var o=new ClipboardJS(".code-copy");o.on("success",o=>showTooltip(o,"Copied!")),o.on("error",o=>showTooltip(o,"Failed..."))});</script>
|
|
<script async="" src="https://cdn.jsdelivr.net/npm/clipboard@2.0.11/dist/clipboard.min.js"></script>
|
|
|
|
|
|
<script src="https://wcpstatic.microsoft.com/mscc/lib/v2/wcp-consent.js" type="text/javascript"></script>
|
|
<script>
|
|
function onConsentChanged(categoryPreferences) {
|
|
console.log("onConsentChanged", categoryPreferences);
|
|
}
|
|
|
|
var siteConsent
|
|
|
|
function initialize(){
|
|
var currentYear = new Date().getFullYear()
|
|
document.getElementById("copyright").innerHTML = `©️ ${currentYear} Microsoft`;
|
|
window.WcpConsent && WcpConsent.init("en-US", "cookie-banner", function (err, _siteConsent) {
|
|
if (!err) {
|
|
siteConsent = _siteConsent; //siteConsent is used to get the current consent
|
|
} else {
|
|
console.log("Error initializing WcpConsent: "+ err);
|
|
}
|
|
}, onConsentChanged, WcpConsent.themes.light);
|
|
}
|
|
|
|
addEventListener("DOMContentLoaded", initialize)
|
|
addEventListener("DOMContentLoaded", checkCookieManager)
|
|
|
|
function checkCookieManager(){
|
|
if(siteConsent.isConsentRequired){
|
|
document.getElementById("cookiesManager").style.display = 'block';
|
|
document.getElementById("divider").style.display = 'block';
|
|
}
|
|
else{
|
|
document.getElementById("cookiesManager").style.display = 'none';
|
|
document.getElementById("divider").style.display = 'none';
|
|
}
|
|
}
|
|
|
|
function manageConsent() {
|
|
if(siteConsent.isConsentRequired){
|
|
siteConsent.manageConsent();
|
|
}
|
|
}
|
|
</script>
|
|
|
|
</head>
|
|
<body>
|
|
<header>
|
|
<div id="cookie-banner"></div>
|
|
<a href="/graphrag/"><span class="logotitle">GraphRAG</span></a>
|
|
</header>
|
|
<div class="page-content">
|
|
<!-- Sidebar -->
|
|
<aside class="menu">
|
|
<ul class="menu-list">
|
|
<li>
|
|
|
|
<a href="/graphrag/">Welcome</a>
|
|
|
|
</li>
|
|
|
|
<!-- Get Started Links -->
|
|
<li>
|
|
|
|
<a href="/graphrag/posts/get_started/">Get Started</a>
|
|
|
|
|
|
<a href="/graphrag/posts/developing/" class="is-active" aria-current="page">Developing</a>
|
|
|
|
</li>
|
|
|
|
<!-- Indexing Links -->
|
|
<li>
|
|
|
|
<a href="/graphrag/posts/index/overview/">Indexing</a>
|
|
|
|
<ul><li>
|
|
<a href="/graphrag/posts/index/0-architecture/">Architecture</a>
|
|
</li><li>
|
|
<a href="/graphrag/posts/index/1-default_dataflow/">Dataflow</a>
|
|
</li><li>
|
|
<a href="/graphrag/posts/index/2-cli/">CLI</a>
|
|
</li><li>
|
|
<a href="/graphrag/posts/index/3-prompt_tuning/">Prompt Tuning</a>
|
|
</li><li>
|
|
|
|
<a href="/graphrag/posts/config/overview/">Configuration</a>
|
|
|
|
<ul>
|
|
<li>
|
|
<a href="/graphrag/posts/config/env_vars">Using Env Vars</a>
|
|
</li>
|
|
<li>
|
|
<a href="/graphrag/posts/config/json_yaml">Using JSON or YAML</a>
|
|
</li>
|
|
<li>
|
|
<a href="/graphrag/posts/config/custom">Fully Custom</a>
|
|
</li>
|
|
<li>
|
|
<a href="/graphrag/posts/config/template">Template</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
|
|
|
|
<!-- Query Links -->
|
|
<li>
|
|
|
|
<a href="/graphrag/posts/query/overview/">Query</a>
|
|
|
|
<ul><li>
|
|
<a href="/graphrag/posts/query/0-global_search/">Global Search</a>
|
|
</li><li>
|
|
<a href="/graphrag/posts/query/1-local_search/">Local Search</a>
|
|
</li><li>
|
|
<a href="/graphrag/posts/query/2-question_generation/">Question Generation</a>
|
|
</li><li>
|
|
<a href="/graphrag/posts/query/3-cli/">CLI</a>
|
|
</li><li>
|
|
|
|
<a href="/graphrag/posts/query/notebooks/overview/">Notebooks</a>
|
|
|
|
<ul>
|
|
<li>
|
|
<a href="/graphrag/posts/query/notebooks/global_search_nb">Global Search</a>
|
|
</li>
|
|
<li>
|
|
<a href="/graphrag/posts/query/notebooks/local_search_nb">Local Search</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</aside>
|
|
|
|
<!-- Main Content -->
|
|
<main>
|
|
<h1>Developing GraphRAG</h1>
|
|
<h1>Requirements</h1>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Name</th>
|
|
<th>Installation</th>
|
|
<th>Purpose</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>Python 3.10 or 3.11</td>
|
|
<td><a href="https://www.python.org/downloads/">Download</a></td>
|
|
<td>The library is Python-based.</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Poetry</td>
|
|
<td><a href="https://python-poetry.org/docs/#installation">Instructions</a></td>
|
|
<td>Poetry is used for package management and virtualenv management in Python codebases</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<h1>Getting Started</h1>
|
|
<h2>Install Dependencies</h2>
|
|
|
|
<div style="position: relative">
|
|
<pre class="language-sh"><code id="code-48" class="language-sh"><span class="token comment"># Install Python dependencies.</span>
|
|
poetry <span class="token function">install</span></code></pre>
|
|
|
|
<button class="code-copy " data-clipboard-target="#code-48" style="position: absolute; top: 7.5px; right: 6px; padding-top: 3px; cursor: pointer; outline: none; opacity: 0.8;" title="Copy">
|
|
<span style="display:inline-block;background:url(https://api.iconify.design/mdi/content-copy.svg) no-repeat center center / contain;width: 16px; height: 16px;" class=""></span>
|
|
</button>
|
|
</div>
|
|
<h2>Execute the Indexing Engine</h2>
|
|
|
|
<div style="position: relative">
|
|
<pre class="language-sh"><code id="code-52" class="language-sh">poetry run poe index <span class="token operator"><</span><span class="token punctuation">..</span>.args<span class="token operator">></span></code></pre>
|
|
|
|
<button class="code-copy " data-clipboard-target="#code-52" style="position: absolute; top: 7.5px; right: 6px; padding-top: 3px; cursor: pointer; outline: none; opacity: 0.8;" title="Copy">
|
|
<span style="display:inline-block;background:url(https://api.iconify.design/mdi/content-copy.svg) no-repeat center center / contain;width: 16px; height: 16px;" class=""></span>
|
|
</button>
|
|
</div>
|
|
<h2>Executing Queries</h2>
|
|
|
|
<div style="position: relative">
|
|
<pre class="language-sh"><code id="code-56" class="language-sh">poetry run poe query <span class="token operator"><</span><span class="token punctuation">..</span>.args<span class="token operator">></span></code></pre>
|
|
|
|
<button class="code-copy " data-clipboard-target="#code-56" style="position: absolute; top: 7.5px; right: 6px; padding-top: 3px; cursor: pointer; outline: none; opacity: 0.8;" title="Copy">
|
|
<span style="display:inline-block;background:url(https://api.iconify.design/mdi/content-copy.svg) no-repeat center center / contain;width: 16px; height: 16px;" class=""></span>
|
|
</button>
|
|
</div>
|
|
<h1>Azurite</h1>
|
|
<p>Some unit and smoke tests use Azurite to emulate Azure resources. This can be started by running:</p>
|
|
|
|
<div style="position: relative">
|
|
<pre class="language-sh"><code id="code-63" class="language-sh">./scripts/start-azurite.sh</code></pre>
|
|
|
|
<button class="code-copy " data-clipboard-target="#code-63" style="position: absolute; top: 7.5px; right: 6px; padding-top: 3px; cursor: pointer; outline: none; opacity: 0.8;" title="Copy">
|
|
<span style="display:inline-block;background:url(https://api.iconify.design/mdi/content-copy.svg) no-repeat center center / contain;width: 16px; height: 16px;" class=""></span>
|
|
</button>
|
|
</div>
|
|
<p>or by simply running <code>azurite</code> in the terminal if already installed globally. See the <a href="https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite">Azurite documentation</a> for more information about how to install and use Azurite.</p>
|
|
<h1>Lifecycle Scripts</h1>
|
|
<p>Our Python package utilizes Poetry to manage dependencies and <a href="https://pypi.org/project/poethepoet/">poethepoet</a> to manage build scripts.</p>
|
|
<p>Available scripts are:</p>
|
|
<ul>
|
|
<li><code>poetry run poe index</code> - Run the Indexing CLI</li>
|
|
<li><code>poetry run poe query</code> - Run the Query CLI</li>
|
|
<li><code>poetry build</code> - This invokes <code>poetry build</code>, which will build a wheel file and other distributable artifacts.</li>
|
|
<li><code>poetry run poe test</code> - This will execute all tests.</li>
|
|
<li><code>poetry run poe test_unit</code> - This will execute unit tests.</li>
|
|
<li><code>poetry run poe test_integration</code> - This will execute integration tests.</li>
|
|
<li><code>poetry run poe test_smoke</code> - This will execute smoke tests.</li>
|
|
<li><code>poetry run poe check</code> - This will perform a suite of static checks across the package, including:
|
|
<ul>
|
|
<li>formatting</li>
|
|
<li>documentation formatting</li>
|
|
<li>linting</li>
|
|
<li>security patterns</li>
|
|
<li>type-checking</li>
|
|
</ul>
|
|
</li>
|
|
<li><code>poetry run poe fix</code> - This will apply any available auto-fixes to the package. Usually this is just formatting fixes.</li>
|
|
<li><code>poetry run poe fix_unsafe</code> - This will apply any available auto-fixes to the package, including those that may be unsafe.</li>
|
|
<li><code>poetry run poe format</code> - Explicitly run the formatter across the package.</li>
|
|
</ul>
|
|
<h2>Troubleshooting</h2>
|
|
<h3>"RuntimeError: llvm-config failed executing, please point LLVM_CONFIG to the path for llvm-config" when running poetry install</h3>
|
|
<p>Make sure llvm-9 and llvm-9-dev are installed:</p>
|
|
<p><code>sudo apt-get install llvm-9 llvm-9-dev</code></p>
|
|
<p>and then in your bashrc, add</p>
|
|
<p><code>export LLVM_CONFIG=/usr/bin/llvm-config-9</code></p>
|
|
<h3>"numba/_pymodule.h:6:10: fatal error: Python.h: No such file or directory" when running poetry install</h3>
|
|
<p>Make sure you have python3.10-dev installed or more generally <code>python<version>-dev</code></p>
|
|
<p><code>sudo apt-get install python3.10-dev</code></p>
|
|
<h3>LLM call constantly exceeds TPM, RPM or time limits</h3>
|
|
<p><code>GRAPHRAG_LLM_THREAD_COUNT</code> and <code>GRAPHRAG_EMBEDDING_THREAD_COUNT</code> are both set to 50 by default. You can modify this values
|
|
to reduce concurrency. Please refer to the <a href="../config/overview">Configuration Documents</a></p>
|
|
|
|
</main>
|
|
</div>
|
|
<footer>
|
|
<a href="https://go.microsoft.com/fwlink/?LinkId=521839">Privacy</a>
|
|
|
|
|
<a href="https://go.microsoft.com/fwlink/?LinkId=2259814">Consumer Health Privacy</a>
|
|
|
|
|
<span id="cookiesManager" onClick="manageConsent();">Cookies</span>
|
|
<span id="divider">|</span>
|
|
<a href="https://go.microsoft.com/fwlink/?LinkID=206977">Terms of Use</a>
|
|
|
|
|
<a href="https://www.microsoft.com/trademarks">Trademarks</a>
|
|
|
|
|
<a href="https://www.microsoft.com" id="copyright"></a>
|
|
|
|
|
<a href="https://github.com/microsoft/graphrag">GitHub</a>
|
|
</footer>
|
|
</body>
|
|
</html> |