llama-hub/tests/test_github_reader.py

from typing import List, Tuple
import unittest
from unittest.mock import MagicMock
import pytest
from unittest.mock import AsyncMock
 
# Skip by default due to network request.
# Remove this to test changes to GithubRepositoryReader.
pytest.skip()

from  loader_hub.github_repo import GithubRepositoryReader, GithubClient

from gpt_index import Document

import os
import pytest
import base64

from loader_hub.github_repo import GithubClient


@pytest.fixture
def github_client():
    return GithubClient(
            github_token=os.getenv("GITHUB_API_TOKEN"),
            verbose= True,
        )


@pytest.mark.asyncio
async def test_github_client(github_client):
    owner = "emptycrown"
    repo = "llama-hub"
    branch = "main"
    commit_sha = "0cd691322e5244b48b68e3588d1343eb53f3a112" # Points to Add spotify reader, https://github.com/emptycrown/llama-hub/commit/0cd691322e5244b48b68e3588d1343eb53f3a112

    # test get_branch
    branch_data = await github_client.get_branch(owner, repo, branch)
    assert branch_data.name == branch
    assert branch_data._links.self == f"https://api.github.com/repos/{owner}/{repo}/branches/{branch}", "Branch self link is incorrect"
    assert branch_data._links.html == f"https://github.com/{owner}/{repo}/tree/{branch}", "Branch html link is incorrect"

    # test get_commit
    commit_data = await github_client.get_commit(owner, repo, commit_sha)
    assert commit_data.sha == commit_sha, "Commit sha is incorrect"
    assert commit_data.url == f"https://api.github.com/repos/{owner}/{repo}/commits/{commit_sha}", "Commit url is incorrect"

    # test get_tree
    tree_data = await github_client.get_tree(owner, repo, commit_data.commit.tree.sha)
    assert tree_data.url == f"https://api.github.com/repos/{owner}/{repo}/git/trees/{commit_data.commit.tree.sha}", "Tree url is incorrect"
    assert tree_data.sha == commit_data.commit.tree.sha, "Tree sha is incorrect"
    print(tree_data.tree[0].sha)
    assert 1 == 1

    # test get_blob
    expected_files_in_first_depth_of_the_tree: List[Tuple[str, str]] = [
        ("test_requirements.txt", "blob"), 
        ("README.md", "blob"), 
        ("Makefile", "blob"), 
        (".gitignore", "blob"),
        ("tests", "tree"),
        ("loader_hub", "tree"),
        (".github", "tree"),
    ]
    # check if the first depth of the tree has the expected files. All the expected files should be in the first depth of the tree and vice versa
    assert len(tree_data.tree) == len(expected_files_in_first_depth_of_the_tree), "The number of files in the first depth of the tree is incorrect"
    for file in expected_files_in_first_depth_of_the_tree:
        assert file in [(tree_file.path, tree_file.type) for tree_file in tree_data.tree], f"{file} is not in the first depth of the tree"
    # checking the opposite
    for tree_obj in tree_data.tree:
        assert (tree_obj.path, tree_obj.type) in expected_files_in_first_depth_of_the_tree, f"{tree_obj.path} is not in the expected files"

    # find test_reqirements.txt in the tree
    test_requirements_txt = [tree_obj for tree_obj in tree_data.tree if tree_obj.path == "test_requirements.txt"][0]

    # test get_blob
    blob_data = await github_client.get_blob(owner, repo, test_requirements_txt.sha)
    assert blob_data.encoding == "base64", "Blob encoding is incorrect"
    assert blob_data.url == f"https://api.github.com/repos/{owner}/{repo}/git/blobs/{test_requirements_txt.sha}", "Blob url is incorrect"
    assert blob_data.sha == test_requirements_txt.sha, "Blob sha is incorrect"
    
    # decode blob content base64-decoded string to utf-8
    decoded_blob_content = base64.b64decode(blob_data.content).decode("utf-8")

    expected_decoded_blob_content = """

# For testing
pytest==7.2.1
pytest-dotenv==0.5.2
# TODO: remove gpt_index after migration
https://github.com/jerryjliu/gpt_index/archive/master.zip

llama-index

# For linting
# linting stubs
types-requests==2.28.11.8
# formatting
black==22.12.0
isort==5.11.4
"""
    # check if the decoded blob content is correct
    for dbc in zip(filter( lambda x: x != "", decoded_blob_content.splitlines()), filter( lambda x: x != "", expected_decoded_blob_content.splitlines())):
        assert dbc[0] == dbc[1], f"{dbc[0]} is not equal to {dbc[1]}"