{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright (c) 2024 Microsoft Corporation.\n",
    "# Licensed under the MIT License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import tiktoken\n",
    "\n",
    "from graphrag.query.context_builder.entity_extraction import EntityVectorStoreKey\n",
    "from graphrag.query.indexer_adapters import (\n",
    "    read_indexer_covariates,\n",
    "    read_indexer_entities,\n",
    "    read_indexer_relationships,\n",
    "    read_indexer_reports,\n",
    "    read_indexer_text_units,\n",
    ")\n",
    "from graphrag.query.llm.oai.chat_openai import ChatOpenAI\n",
    "from graphrag.query.llm.oai.embedding import OpenAIEmbedding\n",
    "from graphrag.query.llm.oai.typing import OpenaiApiType\n",
    "from graphrag.query.question_gen.local_gen import LocalQuestionGen\n",
    "from graphrag.query.structured_search.local_search.mixed_context import (\n",
    "    LocalSearchMixedContext,\n",
    ")\n",
    "from graphrag.query.structured_search.local_search.search import LocalSearch\n",
    "from graphrag.vector_stores.lancedb import LanceDBVectorStore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Local Search Example\n",
    "\n",
    "Local search method generates answers by combining relevant data from the AI-extracted knowledge-graph with text chunks of the raw documents. This method is suitable for questions that require an understanding of specific entities mentioned in the documents (e.g. What are the healing properties of chamomile?)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load text units and graph data tables as context for local search\n",
    "\n",
    "- In this test we first load indexing outputs from parquet files to dataframes, then convert these dataframes into collections of data objects aligning with the knowledge model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load tables to dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIR = \"./inputs/operation dulce\"\n",
    "LANCEDB_URI = f\"{INPUT_DIR}/lancedb\"\n",
    "\n",
    "COMMUNITY_REPORT_TABLE = \"community_reports\"\n",
    "ENTITY_TABLE = \"entities\"\n",
    "COMMUNITY_TABLE = \"communities\"\n",
    "RELATIONSHIP_TABLE = \"relationships\"\n",
    "COVARIATE_TABLE = \"covariates\"\n",
    "TEXT_UNIT_TABLE = \"text_units\"\n",
    "COMMUNITY_LEVEL = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Entity count: 178\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>level</th>\n",
       "      <th>title</th>\n",
       "      <th>type</th>\n",
       "      <th>description</th>\n",
       "      <th>source_id</th>\n",
       "      <th>community</th>\n",
       "      <th>degree</th>\n",
       "      <th>human_readable_id</th>\n",
       "      <th>id</th>\n",
       "      <th>size</th>\n",
       "      <th>graph_embedding</th>\n",
       "      <th>top_level_node_id</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>ALEX MERCER</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>Alex Mercer is a commanding and strategic figu...</td>\n",
       "      <td>06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...</td>\n",
       "      <td>3</td>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>b45241d70f0e43fca764df95b2b81f77</td>\n",
       "      <td>57</td>\n",
       "      <td>None</td>\n",
       "      <td>b45241d70f0e43fca764df95b2b81f77</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>TAYLOR CRUZ</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>Taylor Cruz is a central and commanding figure...</td>\n",
       "      <td>06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...</td>\n",
       "      <td>3</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "      <td>4119fd06010c494caa07f439b333f4c5</td>\n",
       "      <td>52</td>\n",
       "      <td>None</td>\n",
       "      <td>4119fd06010c494caa07f439b333f4c5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>JORDAN HAYES</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>Dr. Jordan Hayes is a central figure at Dulce ...</td>\n",
       "      <td>06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...</td>\n",
       "      <td>3</td>\n",
       "      <td>48</td>\n",
       "      <td>2</td>\n",
       "      <td>d3835bf3dda84ead99deadbeac5d0d7d</td>\n",
       "      <td>48</td>\n",
       "      <td>None</td>\n",
       "      <td>d3835bf3dda84ead99deadbeac5d0d7d</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>SAM RIVERA</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>Sam Rivera is a key operator and technologist ...</td>\n",
       "      <td>06bdff339c02ab35c80fa49320d5da66,1b603cdb97651...</td>\n",
       "      <td>3</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>077d2820ae1845bcbb1803379a3d1eae</td>\n",
       "      <td>48</td>\n",
       "      <td>None</td>\n",
       "      <td>077d2820ae1845bcbb1803379a3d1eae</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>PARANORMAL MILITARY SQUAD</td>\n",
       "      <td>ORGANIZATION</td>\n",
       "      <td>The Paranormal Military Squad is a specialized...</td>\n",
       "      <td>06bdff339c02ab35c80fa49320d5da66,2db9206de77cf...</td>\n",
       "      <td>3</td>\n",
       "      <td>48</td>\n",
       "      <td>4</td>\n",
       "      <td>3671ea0dd4e84c1a9b02c5ab2c8f4bac</td>\n",
       "      <td>48</td>\n",
       "      <td>None</td>\n",
       "      <td>3671ea0dd4e84c1a9b02c5ab2c8f4bac</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   level                      title          type  \\\n",
       "0      0                ALEX MERCER        PERSON   \n",
       "1      0                TAYLOR CRUZ        PERSON   \n",
       "2      0               JORDAN HAYES        PERSON   \n",
       "3      0                 SAM RIVERA        PERSON   \n",
       "4      0  PARANORMAL MILITARY SQUAD  ORGANIZATION   \n",
       "\n",
       "                                         description  \\\n",
       "0  Alex Mercer is a commanding and strategic figu...   \n",
       "1  Taylor Cruz is a central and commanding figure...   \n",
       "2  Dr. Jordan Hayes is a central figure at Dulce ...   \n",
       "3  Sam Rivera is a key operator and technologist ...   \n",
       "4  The Paranormal Military Squad is a specialized...   \n",
       "\n",
       "                                           source_id community  degree  \\\n",
       "0  06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...         3      57   \n",
       "1  06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...         3      52   \n",
       "2  06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...         3      48   \n",
       "3  06bdff339c02ab35c80fa49320d5da66,1b603cdb97651...         3      48   \n",
       "4  06bdff339c02ab35c80fa49320d5da66,2db9206de77cf...         3      48   \n",
       "\n",
       "   human_readable_id                                id  size graph_embedding  \\\n",
       "0                  0  b45241d70f0e43fca764df95b2b81f77    57            None   \n",
       "1                  1  4119fd06010c494caa07f439b333f4c5    52            None   \n",
       "2                  2  d3835bf3dda84ead99deadbeac5d0d7d    48            None   \n",
       "3                  3  077d2820ae1845bcbb1803379a3d1eae    48            None   \n",
       "4                  4  3671ea0dd4e84c1a9b02c5ab2c8f4bac    48            None   \n",
       "\n",
       "                  top_level_node_id  x  y  \n",
       "0  b45241d70f0e43fca764df95b2b81f77  0  0  \n",
       "1  4119fd06010c494caa07f439b333f4c5  0  0  \n",
       "2  d3835bf3dda84ead99deadbeac5d0d7d  0  0  \n",
       "3  077d2820ae1845bcbb1803379a3d1eae  0  0  \n",
       "4  3671ea0dd4e84c1a9b02c5ab2c8f4bac  0  0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read nodes table to get community and degree data\n",
    "entity_df = pd.read_parquet(f\"{INPUT_DIR}/{ENTITY_TABLE}.parquet\")\n",
    "community_df = pd.read_parquet(f\"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet\")\n",
    "\n",
    "entities = read_indexer_entities(entity_df, community_df, COMMUNITY_LEVEL)\n",
    "\n",
    "# load description embeddings to an in-memory lancedb vectorstore\n",
    "# to connect to a remote db, specify url and port values.\n",
    "description_embedding_store = LanceDBVectorStore(\n",
    "    collection_name=\"default-entity-description\",\n",
    ")\n",
    "description_embedding_store.connect(db_uri=LANCEDB_URI)\n",
    "\n",
    "print(f\"Entity count: {len(entity_df)}\")\n",
    "entity_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read relationships"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Relationship count: 373\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>weight</th>\n",
       "      <th>description</th>\n",
       "      <th>text_unit_ids</th>\n",
       "      <th>id</th>\n",
       "      <th>human_readable_id</th>\n",
       "      <th>source_degree</th>\n",
       "      <th>target_degree</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ALEX MERCER</td>\n",
       "      <td>TAYLOR CRUZ</td>\n",
       "      <td>21.0</td>\n",
       "      <td>Alex Mercer and Taylor Cruz are integral membe...</td>\n",
       "      <td>[06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...</td>\n",
       "      <td>148fffeb994541b2b4b6dcefda7001a8</td>\n",
       "      <td>0</td>\n",
       "      <td>57</td>\n",
       "      <td>52</td>\n",
       "      <td>109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ALEX MERCER</td>\n",
       "      <td>JORDAN HAYES</td>\n",
       "      <td>25.0</td>\n",
       "      <td>Alex Mercer and Jordan Hayes are integral team...</td>\n",
       "      <td>[06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...</td>\n",
       "      <td>89c08e793298442686292454a1abff31</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>48</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ALEX MERCER</td>\n",
       "      <td>SAM RIVERA</td>\n",
       "      <td>20.0</td>\n",
       "      <td>Alex Mercer and Sam Rivera are integral member...</td>\n",
       "      <td>[06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...</td>\n",
       "      <td>0467928aa65e4a4fba62bdb1467e3a54</td>\n",
       "      <td>2</td>\n",
       "      <td>57</td>\n",
       "      <td>48</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ALEX MERCER</td>\n",
       "      <td>OPERATION: DULCE BRIEFING ROOM</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Alex Mercer was present in the Operation: Dulc...</td>\n",
       "      <td>[06bdff339c02ab35c80fa49320d5da66]</td>\n",
       "      <td>43c3390303c6476cb65f584e37c3e81c</td>\n",
       "      <td>3</td>\n",
       "      <td>57</td>\n",
       "      <td>4</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ALEX MERCER</td>\n",
       "      <td>PARANORMAL MILITARY SQUAD</td>\n",
       "      <td>20.0</td>\n",
       "      <td>Alex Mercer is a prominent and influential lea...</td>\n",
       "      <td>[06bdff339c02ab35c80fa49320d5da66, 2db9206de77...</td>\n",
       "      <td>fa14b16c17e3417dba5a4b473ea5b18d</td>\n",
       "      <td>4</td>\n",
       "      <td>57</td>\n",
       "      <td>48</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        source                          target  weight  \\\n",
       "0  ALEX MERCER                     TAYLOR CRUZ    21.0   \n",
       "1  ALEX MERCER                    JORDAN HAYES    25.0   \n",
       "2  ALEX MERCER                      SAM RIVERA    20.0   \n",
       "3  ALEX MERCER  OPERATION: DULCE BRIEFING ROOM     1.0   \n",
       "4  ALEX MERCER       PARANORMAL MILITARY SQUAD    20.0   \n",
       "\n",
       "                                         description  \\\n",
       "0  Alex Mercer and Taylor Cruz are integral membe...   \n",
       "1  Alex Mercer and Jordan Hayes are integral team...   \n",
       "2  Alex Mercer and Sam Rivera are integral member...   \n",
       "3  Alex Mercer was present in the Operation: Dulc...   \n",
       "4  Alex Mercer is a prominent and influential lea...   \n",
       "\n",
       "                                       text_unit_ids  \\\n",
       "0  [06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...   \n",
       "1  [06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...   \n",
       "2  [06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...   \n",
       "3                 [06bdff339c02ab35c80fa49320d5da66]   \n",
       "4  [06bdff339c02ab35c80fa49320d5da66, 2db9206de77...   \n",
       "\n",
       "                                 id human_readable_id  source_degree  \\\n",
       "0  148fffeb994541b2b4b6dcefda7001a8                 0             57   \n",
       "1  89c08e793298442686292454a1abff31                 1             57   \n",
       "2  0467928aa65e4a4fba62bdb1467e3a54                 2             57   \n",
       "3  43c3390303c6476cb65f584e37c3e81c                 3             57   \n",
       "4  fa14b16c17e3417dba5a4b473ea5b18d                 4             57   \n",
       "\n",
       "   target_degree  rank  \n",
       "0             52   109  \n",
       "1             48   105  \n",
       "2             48   105  \n",
       "3              4    61  \n",
       "4             48   105  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relationship_df = pd.read_parquet(f\"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet\")\n",
    "relationships = read_indexer_relationships(relationship_df)\n",
    "\n",
    "print(f\"Relationship count: {len(relationship_df)}\")\n",
    "relationship_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Claim records: 156\n"
     ]
    }
   ],
   "source": [
    "# NOTE: covariates are turned off by default, because they generally need prompt tuning to be valuable\n",
    "# Please see the GRAPHRAG_CLAIM_* settings\n",
    "covariate_df = pd.read_parquet(f\"{INPUT_DIR}/{COVARIATE_TABLE}.parquet\")\n",
    "\n",
    "claims = read_indexer_covariates(covariate_df)\n",
    "\n",
    "print(f\"Claim records: {len(claims)}\")\n",
    "covariates = {\"claims\": claims}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read community reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Report records: 20\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>community</th>\n",
       "      <th>full_content</th>\n",
       "      <th>level</th>\n",
       "      <th>rank</th>\n",
       "      <th>title</th>\n",
       "      <th>rank_explanation</th>\n",
       "      <th>summary</th>\n",
       "      <th>findings</th>\n",
       "      <th>full_content_json</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td># Paranormal Military Squad at Dulce Base: Dec...</td>\n",
       "      <td>1</td>\n",
       "      <td>8.5</td>\n",
       "      <td>Paranormal Military Squad at Dulce Base: Decod...</td>\n",
       "      <td>The impact severity rating is high due to the ...</td>\n",
       "      <td>The Paranormal Military Squad, stationed at Du...</td>\n",
       "      <td>[{'explanation': 'Jordan is a central figure i...</td>\n",
       "      <td>{\\n    \"title\": \"Paranormal Military Squad at ...</td>\n",
       "      <td>1ba2d200-dd26-4693-affe-a5539d0a0e0d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td># Dulce and Paranormal Military Squad Operatio...</td>\n",
       "      <td>1</td>\n",
       "      <td>8.5</td>\n",
       "      <td>Dulce and Paranormal Military Squad Operations</td>\n",
       "      <td>The impact severity rating is high due to the ...</td>\n",
       "      <td>The community centers around Dulce, a secretiv...</td>\n",
       "      <td>[{'explanation': 'Dulce is described as a top-...</td>\n",
       "      <td>{\\n    \"title\": \"Dulce and Paranormal Military...</td>\n",
       "      <td>a8a530b0-ae6b-44ea-b11c-9f70d138298d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td># Paranormal Military Squad and Dulce Base Ope...</td>\n",
       "      <td>1</td>\n",
       "      <td>7.5</td>\n",
       "      <td>Paranormal Military Squad and Dulce Base Opera...</td>\n",
       "      <td>The impact severity rating is relatively high ...</td>\n",
       "      <td>The community centers around the Paranormal Mi...</td>\n",
       "      <td>[{'explanation': 'Taylor is a central figure w...</td>\n",
       "      <td>{\\n    \"title\": \"Paranormal Military Squad and...</td>\n",
       "      <td>0478975b-c805-4cc1-b746-82f3e689e2f3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13</td>\n",
       "      <td># Mission Dynamics and Leadership: Cruz and Wa...</td>\n",
       "      <td>1</td>\n",
       "      <td>7.5</td>\n",
       "      <td>Mission Dynamics and Leadership: Cruz and Wash...</td>\n",
       "      <td>The impact severity rating is relatively high ...</td>\n",
       "      <td>This report explores the intricate dynamics of...</td>\n",
       "      <td>[{'explanation': 'Cruz is a central figure in ...</td>\n",
       "      <td>{\\n    \"title\": \"Mission Dynamics and Leadersh...</td>\n",
       "      <td>b56f6e68-3951-4f07-8760-63700944a375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14</td>\n",
       "      <td># Dulce Base and Paranormal Military Squad: Br...</td>\n",
       "      <td>1</td>\n",
       "      <td>8.5</td>\n",
       "      <td>Dulce Base and Paranormal Military Squad: Brid...</td>\n",
       "      <td>The impact severity rating is high due to the ...</td>\n",
       "      <td>The community centers around the Dulce Base, a...</td>\n",
       "      <td>[{'explanation': 'Sam Rivera, a member of the ...</td>\n",
       "      <td>{\\n    \"title\": \"Dulce Base and Paranormal Mil...</td>\n",
       "      <td>736e7006-d050-4abb-a122-00febf3f540f</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  community                                       full_content  level  rank  \\\n",
       "0        10  # Paranormal Military Squad at Dulce Base: Dec...      1   8.5   \n",
       "1        11  # Dulce and Paranormal Military Squad Operatio...      1   8.5   \n",
       "2        12  # Paranormal Military Squad and Dulce Base Ope...      1   7.5   \n",
       "3        13  # Mission Dynamics and Leadership: Cruz and Wa...      1   7.5   \n",
       "4        14  # Dulce Base and Paranormal Military Squad: Br...      1   8.5   \n",
       "\n",
       "                                               title  \\\n",
       "0  Paranormal Military Squad at Dulce Base: Decod...   \n",
       "1     Dulce and Paranormal Military Squad Operations   \n",
       "2  Paranormal Military Squad and Dulce Base Opera...   \n",
       "3  Mission Dynamics and Leadership: Cruz and Wash...   \n",
       "4  Dulce Base and Paranormal Military Squad: Brid...   \n",
       "\n",
       "                                    rank_explanation  \\\n",
       "0  The impact severity rating is high due to the ...   \n",
       "1  The impact severity rating is high due to the ...   \n",
       "2  The impact severity rating is relatively high ...   \n",
       "3  The impact severity rating is relatively high ...   \n",
       "4  The impact severity rating is high due to the ...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  The Paranormal Military Squad, stationed at Du...   \n",
       "1  The community centers around Dulce, a secretiv...   \n",
       "2  The community centers around the Paranormal Mi...   \n",
       "3  This report explores the intricate dynamics of...   \n",
       "4  The community centers around the Dulce Base, a...   \n",
       "\n",
       "                                            findings  \\\n",
       "0  [{'explanation': 'Jordan is a central figure i...   \n",
       "1  [{'explanation': 'Dulce is described as a top-...   \n",
       "2  [{'explanation': 'Taylor is a central figure w...   \n",
       "3  [{'explanation': 'Cruz is a central figure in ...   \n",
       "4  [{'explanation': 'Sam Rivera, a member of the ...   \n",
       "\n",
       "                                   full_content_json  \\\n",
       "0  {\\n    \"title\": \"Paranormal Military Squad at ...   \n",
       "1  {\\n    \"title\": \"Dulce and Paranormal Military...   \n",
       "2  {\\n    \"title\": \"Paranormal Military Squad and...   \n",
       "3  {\\n    \"title\": \"Mission Dynamics and Leadersh...   \n",
       "4  {\\n    \"title\": \"Dulce Base and Paranormal Mil...   \n",
       "\n",
       "                                     id  \n",
       "0  1ba2d200-dd26-4693-affe-a5539d0a0e0d  \n",
       "1  a8a530b0-ae6b-44ea-b11c-9f70d138298d  \n",
       "2  0478975b-c805-4cc1-b746-82f3e689e2f3  \n",
       "3  b56f6e68-3951-4f07-8760-63700944a375  \n",
       "4  736e7006-d050-4abb-a122-00febf3f540f  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report_df = pd.read_parquet(f\"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet\")\n",
    "reports = read_indexer_reports(report_df, community_df, COMMUNITY_LEVEL)\n",
    "\n",
    "print(f\"Report records: {len(report_df)}\")\n",
    "report_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read text units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text unit records: 50\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>n_tokens</th>\n",
       "      <th>document_ids</th>\n",
       "      <th>entity_ids</th>\n",
       "      <th>relationship_ids</th>\n",
       "      <th>covariate_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>06bdff339c02ab35c80fa49320d5da66</td>\n",
       "      <td># Operation: Dulce\\n\\n## Chapter 1\\n\\nThe thru...</td>\n",
       "      <td>600</td>\n",
       "      <td>[958fdd043f17ade63cb13570b59df295]</td>\n",
       "      <td>[b45241d70f0e43fca764df95b2b81f77, 4119fd06010...</td>\n",
       "      <td>[148fffeb994541b2b4b6dcefda7001a8, 89c08e79329...</td>\n",
       "      <td>[439081a3-bfeb-4693-968a-0f6189d8fa50, 3399e3d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>28dc4aa41a3e99deb7f354682f3e9904</td>\n",
       "      <td>'s authoritarian performance. _Protocols_, Jor...</td>\n",
       "      <td>600</td>\n",
       "      <td>[958fdd043f17ade63cb13570b59df295]</td>\n",
       "      <td>[4119fd06010c494caa07f439b333f4c5, 077d2820ae1...</td>\n",
       "      <td>[9a6f414210e14841a5b0e661aedc898d, db541b72609...</td>\n",
       "      <td>[b09594cb-d4b7-4de4-a1af-97778300eb1b, f7c5ea4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>813db3138ef511c34be86f841f68aa8f</td>\n",
       "      <td>, rehearsing the speech for their subordinates...</td>\n",
       "      <td>600</td>\n",
       "      <td>[958fdd043f17ade63cb13570b59df295]</td>\n",
       "      <td>[b45241d70f0e43fca764df95b2b81f77, 077d2820ae1...</td>\n",
       "      <td>[0467928aa65e4a4fba62bdb1467e3a54, 7cc3356d38d...</td>\n",
       "      <td>[7642fe21-7cb9-4428-848b-d2e3f5ab10ca, 6fe3d6c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d0afd106abf3a2966ff88a99eef710db</td>\n",
       "      <td>, weighing his words carefully. \"Our tech is t...</td>\n",
       "      <td>600</td>\n",
       "      <td>[958fdd043f17ade63cb13570b59df295]</td>\n",
       "      <td>[b45241d70f0e43fca764df95b2b81f77, 4119fd06010...</td>\n",
       "      <td>[148fffeb994541b2b4b6dcefda7001a8, 89c08e79329...</td>\n",
       "      <td>[956e4c22-e343-4b5d-ad3c-d44ca3ce5fb5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f4c7c95d7bf621c6eb73d331cbf0b608</td>\n",
       "      <td>stepping into their exchange. The authority i...</td>\n",
       "      <td>600</td>\n",
       "      <td>[958fdd043f17ade63cb13570b59df295]</td>\n",
       "      <td>[077d2820ae1845bcbb1803379a3d1eae, 1fd3fa8bb5a...</td>\n",
       "      <td>[478e4c72d8fb46dd8cc9f0691c9878fd, 82b0446e7c9...</td>\n",
       "      <td>[a54eda4d-b5bf-471d-989f-370eb9aff961, 9f85274...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 id  \\\n",
       "0  06bdff339c02ab35c80fa49320d5da66   \n",
       "1  28dc4aa41a3e99deb7f354682f3e9904   \n",
       "2  813db3138ef511c34be86f841f68aa8f   \n",
       "3  d0afd106abf3a2966ff88a99eef710db   \n",
       "4  f4c7c95d7bf621c6eb73d331cbf0b608   \n",
       "\n",
       "                                                text  n_tokens  \\\n",
       "0  # Operation: Dulce\\n\\n## Chapter 1\\n\\nThe thru...       600   \n",
       "1  's authoritarian performance. _Protocols_, Jor...       600   \n",
       "2  , rehearsing the speech for their subordinates...       600   \n",
       "3  , weighing his words carefully. \"Our tech is t...       600   \n",
       "4   stepping into their exchange. The authority i...       600   \n",
       "\n",
       "                         document_ids  \\\n",
       "0  [958fdd043f17ade63cb13570b59df295]   \n",
       "1  [958fdd043f17ade63cb13570b59df295]   \n",
       "2  [958fdd043f17ade63cb13570b59df295]   \n",
       "3  [958fdd043f17ade63cb13570b59df295]   \n",
       "4  [958fdd043f17ade63cb13570b59df295]   \n",
       "\n",
       "                                          entity_ids  \\\n",
       "0  [b45241d70f0e43fca764df95b2b81f77, 4119fd06010...   \n",
       "1  [4119fd06010c494caa07f439b333f4c5, 077d2820ae1...   \n",
       "2  [b45241d70f0e43fca764df95b2b81f77, 077d2820ae1...   \n",
       "3  [b45241d70f0e43fca764df95b2b81f77, 4119fd06010...   \n",
       "4  [077d2820ae1845bcbb1803379a3d1eae, 1fd3fa8bb5a...   \n",
       "\n",
       "                                    relationship_ids  \\\n",
       "0  [148fffeb994541b2b4b6dcefda7001a8, 89c08e79329...   \n",
       "1  [9a6f414210e14841a5b0e661aedc898d, db541b72609...   \n",
       "2  [0467928aa65e4a4fba62bdb1467e3a54, 7cc3356d38d...   \n",
       "3  [148fffeb994541b2b4b6dcefda7001a8, 89c08e79329...   \n",
       "4  [478e4c72d8fb46dd8cc9f0691c9878fd, 82b0446e7c9...   \n",
       "\n",
       "                                       covariate_ids  \n",
       "0  [439081a3-bfeb-4693-968a-0f6189d8fa50, 3399e3d...  \n",
       "1  [b09594cb-d4b7-4de4-a1af-97778300eb1b, f7c5ea4...  \n",
       "2  [7642fe21-7cb9-4428-848b-d2e3f5ab10ca, 6fe3d6c...  \n",
       "3             [956e4c22-e343-4b5d-ad3c-d44ca3ce5fb5]  \n",
       "4  [a54eda4d-b5bf-471d-989f-370eb9aff961, 9f85274...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_unit_df = pd.read_parquet(f\"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet\")\n",
    "text_units = read_indexer_text_units(text_unit_df)\n",
    "\n",
    "print(f\"Text unit records: {len(text_unit_df)}\")\n",
    "text_unit_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = os.environ[\"GRAPHRAG_API_KEY\"]\n",
    "llm_model = os.environ[\"GRAPHRAG_LLM_MODEL\"]\n",
    "embedding_model = os.environ[\"GRAPHRAG_EMBEDDING_MODEL\"]\n",
    "\n",
    "llm = ChatOpenAI(\n",
    "    api_key=api_key,\n",
    "    model=llm_model,\n",
    "    api_type=OpenaiApiType.OpenAI,  # OpenaiApiType.OpenAI or OpenaiApiType.AzureOpenAI\n",
    "    max_retries=20,\n",
    ")\n",
    "\n",
    "token_encoder = tiktoken.get_encoding(\"cl100k_base\")\n",
    "\n",
    "text_embedder = OpenAIEmbedding(\n",
    "    api_key=api_key,\n",
    "    api_base=None,\n",
    "    api_type=OpenaiApiType.OpenAI,\n",
    "    model=embedding_model,\n",
    "    deployment_name=embedding_model,\n",
    "    max_retries=20,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create local search context builder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "context_builder = LocalSearchMixedContext(\n",
    "    community_reports=reports,\n",
    "    text_units=text_units,\n",
    "    entities=entities,\n",
    "    relationships=relationships,\n",
    "    # if you did not run covariates during indexing, set this to None\n",
    "    covariates=covariates,\n",
    "    entity_text_embeddings=description_embedding_store,\n",
    "    embedding_vectorstore_key=EntityVectorStoreKey.ID,  # if the vectorstore uses entity title as ids, set this to EntityVectorStoreKey.TITLE\n",
    "    text_embedder=text_embedder,\n",
    "    token_encoder=token_encoder,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create local search engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# text_unit_prop: proportion of context window dedicated to related text units\n",
    "# community_prop: proportion of context window dedicated to community reports.\n",
    "# The remaining proportion is dedicated to entities and relationships. Sum of text_unit_prop and community_prop should be <= 1\n",
    "# conversation_history_max_turns: maximum number of turns to include in the conversation history.\n",
    "# conversation_history_user_turns_only: if True, only include user queries in the conversation history.\n",
    "# top_k_mapped_entities: number of related entities to retrieve from the entity description embedding store.\n",
    "# top_k_relationships: control the number of out-of-network relationships to pull into the context window.\n",
    "# include_entity_rank: if True, include the entity rank in the entity table in the context window. Default entity rank = node degree.\n",
    "# include_relationship_weight: if True, include the relationship weight in the context window.\n",
    "# include_community_rank: if True, include the community rank in the context window.\n",
    "# return_candidate_context: if True, return a set of dataframes containing all candidate entity/relationship/covariate records that\n",
    "# could be relevant. Note that not all of these records will be included in the context window. The \"in_context\" column in these\n",
    "# dataframes indicates whether the record is included in the context window.\n",
    "# max_tokens: maximum number of tokens to use for the context window.\n",
    "\n",
    "\n",
    "local_context_params = {\n",
    "    \"text_unit_prop\": 0.5,\n",
    "    \"community_prop\": 0.1,\n",
    "    \"conversation_history_max_turns\": 5,\n",
    "    \"conversation_history_user_turns_only\": True,\n",
    "    \"top_k_mapped_entities\": 10,\n",
    "    \"top_k_relationships\": 10,\n",
    "    \"include_entity_rank\": True,\n",
    "    \"include_relationship_weight\": True,\n",
    "    \"include_community_rank\": False,\n",
    "    \"return_candidate_context\": False,\n",
    "    \"embedding_vectorstore_key\": EntityVectorStoreKey.ID,  # set this to EntityVectorStoreKey.TITLE if the vectorstore uses entity title as ids\n",
    "    \"max_tokens\": 12_000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)\n",
    "}\n",
    "\n",
    "llm_params = {\n",
    "    \"max_tokens\": 2_000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000=1500)\n",
    "    \"temperature\": 0.0,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_engine = LocalSearch(\n",
    "    llm=llm,\n",
    "    context_builder=context_builder,\n",
    "    token_encoder=token_encoder,\n",
    "    llm_params=llm_params,\n",
    "    context_builder_params=local_context_params,\n",
    "    response_type=\"multiple paragraphs\",  # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run local search on sample queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "### Overview of Agent Mercer\n",
      "\n",
      "Agent Alex Mercer is a prominent figure within the Paranormal Military Squad, playing a crucial role in Operation: Dulce at the Dulce Base. He is recognized for his commanding and strategic presence, particularly in overseeing operations related to communication with extraterrestrial intelligence. His responsibilities include initiating broadcasts to communicate with extraterrestrial beings, decoding alien messages, and leading the team in understanding and responding to these communications [Data: Entities (0)].\n",
      "\n",
      "### Role and Responsibilities\n",
      "\n",
      "Mercer is deeply involved in the philosophical and strategic aspects of interstellar communication, viewing these interactions as a form of cosmic dialogue. His leadership style is characterized by a blend of determination, compliance with mission protocols, and a protective approach towards his team. He collaborates with team members like Jordan Hayes, exploring secured areas and engaging in high-stakes, secretive operations. Mercer emphasizes the importance of intuition and trust beyond protocol, particularly in his mentorship of Sam Rivera [Data: Entities (0)].\n",
      "\n",
      "### Relationships and Influence\n",
      "\n",
      "Agent Mercer is known for his intellectual curiosity and deep involvement in the mission's strategic aspects. He is depicted as a thoughtful mentor, particularly to Sam Rivera, and his experiences during encounters with alien signals have led to profound changes, reinforcing his role as a key decision-maker and guardian in missions that transcend traditional boundaries [Data: Entities (0); Relationships (167)].\n",
      "\n",
      "### Involvement in Operation: Dulce\n",
      "\n",
      "Operation: Dulce was a significant mission undertaken by the Paranormal Military Squad, with Mercer being a key participant. The operation involved exploring paranormal anomalies and assessing potential threats, marking the beginning of an interstellar odyssey and leading to the first contact with an alien race. Mercer's involvement in this operation highlights his critical role in expanding the boundaries of human understanding of the universe [Data: Entities (5); Relationships (245)].\n",
      "\n",
      "In summary, Agent Alex Mercer is a pivotal figure in the Paranormal Military Squad, known for his strategic leadership and deep involvement in extraterrestrial communication efforts. His role in Operation: Dulce underscores his importance in the mission to uncover the secrets of the Dulce Base and engage with alien intelligence.\n"
     ]
    }
   ],
   "source": [
    "result = await search_engine.search(\"Tell me about Agent Mercer\")\n",
    "print(result.response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "## Overview of Dr. Jordan Hayes\n",
      "\n",
      "Dr. Jordan Hayes is a prominent scientist at Dulce Base, playing a crucial role in the Paranormal Military Squad's efforts to understand and communicate with extraterrestrial entities. Dr. Hayes's work is primarily focused on decoding and analyzing alien signals and codes, which are essential for interstellar communication and potentially interspecies interaction. This involves decrypting algorithms, interpreting cosmic signals, and analyzing the implications of alien society [Data: Entities (2, 17, 47); Relationships (201, 322, 283)].\n",
      "\n",
      "## Key Contributions and Expertise\n",
      "\n",
      "Dr. Hayes is noted for their adaptability and skepticism, qualities that are essential given the uncertainties and unknown challenges of their mission. They are deeply involved in the scientific exploration aspects of Operation: Dulce, where their efforts are on the verge of a significant scientific breakthrough. Dr. Hayes leads efforts in isolating and understanding complex alien signals that resemble human cognition, suggesting that these signals are artificial and patterned, indicating a tandem evolution with humanity [Data: Entities (2, 17); Claims (60, 83, 91, 134)].\n",
      "\n",
      "## Collaborative Efforts and Leadership\n",
      "\n",
      "Dr. Hayes works closely with colleagues like Alex Mercer and Taylor Cruz, engaging in thoughtful dialogue and showing analytical thinking about the mission's uncertainties. Despite some tension with Taylor Cruz due to differing leadership styles, Dr. Hayes continues to collaborate effectively with the team, contributing to the strategic discussions and decisions made in the briefing room. Their role in the command center at Dulce Base involves setting up lab stations, operating the mainframe, and playing a crucial role in the command center [Data: Entities (2, 17); Relationships (26, 82, 175, 322); Claims (2, 13)].\n",
      "\n",
      "## Scientific Breakthroughs and Challenges\n",
      "\n",
      "Dr. Hayes's work suggests that the alien signals they are studying are not just random but are structured and intentional, potentially indicating a form of extraterrestrial communication. This has led to the identification of new inviting signal patterns suggesting an intelligent confluence. Dr. Hayes's efforts are crucial in crafting humanity's responses to cosmic alignments with stars and responsive galactic signals, navigating the dark corridors of Dulce with a focus on the unknown variables and challenges beyond established protocols [Data: Entities (2, 17); Claims (60, 83, 91, 134, 153)].\n",
      "\n",
      "In summary, Dr. Jordan Hayes is a central figure in the efforts to understand and communicate with extraterrestrial entities at Dulce Base. Their work is characterized by a blend of scientific rigor, adaptability, and collaboration, making them a vital asset to the Paranormal Military Squad's mission.\n"
     ]
    }
   ],
   "source": [
    "question = \"Tell me about Dr. Jordan Hayes\"\n",
    "result = await search_engine.search(question)\n",
    "print(result.response)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inspecting the context data used to generate the response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>entity</th>\n",
       "      <th>description</th>\n",
       "      <th>number of relationships</th>\n",
       "      <th>in_context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17</td>\n",
       "      <td>DR. JORDAN HAYES</td>\n",
       "      <td>Dr. Jordan Hayes is a key scientist at Dulce B...</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>JORDAN</td>\n",
       "      <td>Jordan Hayes is a key member of the Paranormal...</td>\n",
       "      <td>22</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>JORDAN HAYES</td>\n",
       "      <td>Dr. Jordan Hayes is a central figure at Dulce ...</td>\n",
       "      <td>48</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>47</td>\n",
       "      <td>DR. HAYES</td>\n",
       "      <td>Dr. Hayes is a scientist working on understand...</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>AGENT HAYES</td>\n",
       "      <td>Agent Hayes is known for emphasizing empowerme...</td>\n",
       "      <td>4</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id            entity                                        description  \\\n",
       "0  17  DR. JORDAN HAYES  Dr. Jordan Hayes is a key scientist at Dulce B...   \n",
       "1   7            JORDAN  Jordan Hayes is a key member of the Paranormal...   \n",
       "2   2      JORDAN HAYES  Dr. Jordan Hayes is a central figure at Dulce ...   \n",
       "3  47         DR. HAYES  Dr. Hayes is a scientist working on understand...   \n",
       "4  10       AGENT HAYES  Agent Hayes is known for emphasizing empowerme...   \n",
       "\n",
       "  number of relationships  in_context  \n",
       "0                      18        True  \n",
       "1                      22        True  \n",
       "2                      48        True  \n",
       "3                       1        True  \n",
       "4                       4        True  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.context_data[\"entities\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>description</th>\n",
       "      <th>weight</th>\n",
       "      <th>links</th>\n",
       "      <th>in_context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>260</td>\n",
       "      <td>JORDAN</td>\n",
       "      <td>DR. JORDAN HAYES</td>\n",
       "      <td>Dr. Jordan Hayes and Jordan refer to the same ...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>ALEX MERCER</td>\n",
       "      <td>JORDAN</td>\n",
       "      <td>Alex Mercer values Jordan's analytical approac...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>26</td>\n",
       "      <td>ALEX MERCER</td>\n",
       "      <td>DR. JORDAN HAYES</td>\n",
       "      <td>Alex Mercer and Dr. Jordan Hayes are collabora...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>77</td>\n",
       "      <td>TAYLOR CRUZ</td>\n",
       "      <td>JORDAN</td>\n",
       "      <td>Jordan and Taylor Cruz are team members workin...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>82</td>\n",
       "      <td>TAYLOR CRUZ</td>\n",
       "      <td>DR. JORDAN HAYES</td>\n",
       "      <td>Dr. Jordan Hayes and Taylor Cruz are colleague...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id       source            target  \\\n",
       "0  260       JORDAN  DR. JORDAN HAYES   \n",
       "1   50  ALEX MERCER            JORDAN   \n",
       "2   26  ALEX MERCER  DR. JORDAN HAYES   \n",
       "3   77  TAYLOR CRUZ            JORDAN   \n",
       "4   82  TAYLOR CRUZ  DR. JORDAN HAYES   \n",
       "\n",
       "                                         description weight links  in_context  \n",
       "0  Dr. Jordan Hayes and Jordan refer to the same ...    1.0     1        True  \n",
       "1  Alex Mercer values Jordan's analytical approac...    1.0     2        True  \n",
       "2  Alex Mercer and Dr. Jordan Hayes are collabora...    5.0     2        True  \n",
       "3  Jordan and Taylor Cruz are team members workin...    4.0     2        True  \n",
       "4  Dr. Jordan Hayes and Taylor Cruz are colleague...    4.0     2        True  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.context_data[\"relationships\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'reports'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[17], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mresult\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcontext_data\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mreports\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mhead()\n",
      "\u001b[0;31mKeyError\u001b[0m: 'reports'"
     ]
    }
   ],
   "source": [
    "if \"reports\" in result.context_data:\n",
    "    result.context_data[\"reports\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.context_data[\"sources\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"claims\" in result.context_data:\n",
    "    print(result.context_data[\"claims\"].head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This function takes a list of user queries and generates the next candidate questions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "question_generator = LocalQuestionGen(\n",
    "    llm=llm,\n",
    "    context_builder=context_builder,\n",
    "    token_encoder=token_encoder,\n",
    "    llm_params=llm_params,\n",
    "    context_builder_params=local_context_params,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "question_history = [\n",
    "    \"Tell me about Agent Mercer\",\n",
    "    \"What happens in Dulce military base?\",\n",
    "]\n",
    "candidate_questions = await question_generator.agenerate(\n",
    "    question_history=question_history, context_data=None, question_count=5\n",
    ")\n",
    "print(candidate_questions.response)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}