yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-15 03:56:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): add cleanup in tests, make urls configurable (#5287)

Browse Source
This commit is contained in:
Aseem Bansal 2022-06-30 16:00:50 +05:30 committed by GitHub
parent 11356e37ba
commit c95dc926a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 875 additions and 851 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

546
smoke-test/test_e2e.py
View File

File diff suppressed because it is too large Load Diff

30
smoke-test/test_rapid.py
View File

@ -1,18 +1,14 @@
import time import time
import urllib
from typing import Any, Dict, Optional, cast
import pytest import pytest
import requests import requests
from datahub.cli.docker import check_local_docker_containers from datahub.cli.docker import check_local_docker_containers
from datahub.ingestion.run.pipeline import Pipeline
from datahub.ingestion.source.state.checkpoint import Checkpoint from tests.utils import get_frontend_url, ingest_file_via_rest
from tests.utils import ingest_file_via_rest
bootstrap_small = "test_resources/bootstrap_single.json" bootstrap_small = "test_resources/bootstrap_single.json"
bootstrap_small_2 = "test_resources/bootstrap_single2.json" bootstrap_small_2 = "test_resources/bootstrap_single2.json"
FRONTEND_ENDPOINT = "http://localhost:9002"
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def wait_for_healthchecks(): def wait_for_healthchecks():
@ -20,6 +16,7 @@ def wait_for_healthchecks():
assert not check_local_docker_containers() assert not check_local_docker_containers()
yield yield
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def frontend_session(wait_for_healthchecks): def frontend_session(wait_for_healthchecks):
session = requests.Session() session = requests.Session()
@ -28,19 +25,18 @@ def frontend_session(wait_for_healthchecks):
"Content-Type": "application/json", "Content-Type": "application/json",
} }
data = '{"username":"datahub", "password":"datahub"}' data = '{"username":"datahub", "password":"datahub"}'
response = session.post( response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data)
f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data
)
response.raise_for_status() response.raise_for_status()
yield session yield session
def test_ingestion_via_rest_rapid(frontend_session, wait_for_healthchecks): def test_ingestion_via_rest_rapid(frontend_session, wait_for_healthchecks):
ingest_file_via_rest(bootstrap_small) ingest_file_via_rest(bootstrap_small)
ingest_file_via_rest(bootstrap_small_2) ingest_file_via_rest(bootstrap_small_2)
urn = f"urn:li:dataset:(urn:li:dataPlatform:testPlatform,testDataset,PROD)" urn = "urn:li:dataset:(urn:li:dataPlatform:testPlatform,testDataset,PROD)"
json = { json = {
"query": """query getDataset($urn: String!) {\n "query": """query getDataset($urn: String!) {\n
dataset(urn: $urn) {\n dataset(urn: $urn) {\n
urn\n urn\n
name\n name\n
@ -70,15 +66,11 @@ def test_ingestion_via_rest_rapid(frontend_session, wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": urn},
"urn": urn }
}
}
# #
time.sleep(2) time.sleep(2)
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()

18
smoke-test/tests/assertions/assertions_test.py
View File

@ -4,7 +4,6 @@ import urllib
import pytest import pytest
import requests import requests
from datahub.cli.docker import check_local_docker_containers from datahub.cli.docker import check_local_docker_containers
from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp import MetadataChangeProposalWrapper
@ -25,11 +24,7 @@ from datahub.metadata.schema_classes import (
PartitionSpecClass, PartitionSpecClass,
PartitionTypeClass, PartitionTypeClass,
) )
from tests.utils import ingest_file_via_rest from tests.utils import delete_urns_from_file, get_gms_url, ingest_file_via_rest
from tests.utils import delete_urns_from_file
GMS_ENDPOINT = "http://localhost:8080"
restli_default_headers = { restli_default_headers = {
"X-RestLi-Protocol-Version": "2.0.0", "X-RestLi-Protocol-Version": "2.0.0",
@ -253,6 +248,7 @@ def test_healthchecks(wait_for_healthchecks):
def test_run_ingestion(generate_test_data): def test_run_ingestion(generate_test_data):
ingest_file_via_rest(generate_test_data) ingest_file_via_rest(generate_test_data)
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_gms_get_latest_assertions_results_by_partition(): def test_gms_get_latest_assertions_results_by_partition():
urn = make_dataset_urn("postgres", "foo") urn = make_dataset_urn("postgres", "foo")
@ -294,7 +290,7 @@ def test_gms_get_latest_assertions_results_by_partition():
} }
) )
response = requests.post( response = requests.post(
f"{GMS_ENDPOINT}/analytics?action=getTimeseriesStats", f"{get_gms_url()}/analytics?action=getTimeseriesStats",
data=query, data=query,
headers=restli_default_headers, headers=restli_default_headers,
) )
@ -325,7 +321,7 @@ def test_gms_get_assertions_on_dataset():
"""lists all assertion urns including those which may not have executed""" """lists all assertion urns including those which may not have executed"""
urn = make_dataset_urn("postgres", "foo") urn = make_dataset_urn("postgres", "foo")
response = requests.get( response = requests.get(
f"{GMS_ENDPOINT}/relationships?direction=INCOMING&urn={urllib.parse.quote(urn)}&types=Asserts" f"{get_gms_url()}/relationships?direction=INCOMING&urn={urllib.parse.quote(urn)}&types=Asserts"
) )
response.raise_for_status() response.raise_for_status()
@ -339,7 +335,7 @@ def test_gms_get_assertions_on_dataset_field():
dataset_urn = make_dataset_urn("postgres", "foo") dataset_urn = make_dataset_urn("postgres", "foo")
field_urn = make_schema_field_urn(dataset_urn, "col1") field_urn = make_schema_field_urn(dataset_urn, "col1")
response = requests.get( response = requests.get(
f"{GMS_ENDPOINT}/relationships?direction=INCOMING&urn={urllib.parse.quote(field_urn)}&types=Asserts" f"{get_gms_url()}/relationships?direction=INCOMING&urn={urllib.parse.quote(field_urn)}&types=Asserts"
) )
response.raise_for_status() response.raise_for_status()
@ -351,7 +347,7 @@ def test_gms_get_assertions_on_dataset_field():
def test_gms_get_assertion_info(): def test_gms_get_assertion_info():
assertion_urn = "urn:li:assertion:2d3b06a6e77e1f24adc9860a05ea089b" assertion_urn = "urn:li:assertion:2d3b06a6e77e1f24adc9860a05ea089b"
response = requests.get( response = requests.get(
f"{GMS_ENDPOINT}/aspects/{urllib.parse.quote(assertion_urn)}\ f"{get_gms_url()}/aspects/{urllib.parse.quote(assertion_urn)}\
?aspect=assertionInfo&version=0", ?aspect=assertionInfo&version=0",
headers=restli_default_headers, headers=restli_default_headers,
) )
@ -364,4 +360,4 @@ def test_gms_get_assertion_info():
assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["type"] == "DATASET" assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["type"] == "DATASET"
assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["datasetAssertion"][ assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["datasetAssertion"][
"scope" "scope"
] ]

17
smoke-test/tests/cli/datahub_graph_test.py
View File

@ -1,14 +1,7 @@
import time
import pytest import pytest
from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph
from datahub.metadata.schema_classes import SchemaMetadataClass, KafkaSchemaClass from datahub.metadata.schema_classes import KafkaSchemaClass, SchemaMetadataClass
from tests.utils import ( from tests.utils import delete_urns_from_file, ingest_file_via_rest
FRONTEND_ENDPOINT,
GMS_ENDPOINT,
delete_urns_from_file,
ingest_file_via_rest,
)
@pytest.fixture(scope="module", autouse=False) @pytest.fixture(scope="module", autouse=False)
@ -38,5 +31,7 @@ def test_get_aspect_v2(frontend_session, ingest_cleanup_data):
assert schema_metadata.platform == "urn:li:dataPlatform:kafka" assert schema_metadata.platform == "urn:li:dataPlatform:kafka"
assert isinstance(schema_metadata.platformSchema, KafkaSchemaClass) assert isinstance(schema_metadata.platformSchema, KafkaSchemaClass)
k_schema: KafkaSchemaClass = schema_metadata.platformSchema k_schema: KafkaSchemaClass = schema_metadata.platformSchema
assert k_schema.documentSchema == "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" assert (
k_schema.documentSchema
== '{"type":"record","name":"SampleKafkaSchema","namespace":"com.linkedin.dataset","doc":"Sample Kafka dataset","fields":[{"name":"field_foo","type":["string"]},{"name":"field_bar","type":["boolean"]}]}'
)

15
smoke-test/tests/conftest.py
View File

@ -1,22 +1,22 @@
import os import os
import time
import pytest import pytest
import requests import requests
import urllib
from datahub.cli.docker import check_local_docker_containers from datahub.cli.docker import check_local_docker_containers
from datahub.ingestion.run.pipeline import Pipeline
from tests.utils import FRONTEND_ENDPOINT from tests.utils import get_frontend_url
# Disable telemetry # Disable telemetry
os.putenv("DATAHUB_TELEMETRY_ENABLED", "false") os.putenv("DATAHUB_TELEMETRY_ENABLED", "false")
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def wait_for_healthchecks(): def wait_for_healthchecks():
# Simply assert that everything is healthy, but don't wait. # Simply assert that everything is healthy, but don't wait.
assert not check_local_docker_containers() assert not check_local_docker_containers()
yield yield
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def frontend_session(wait_for_healthchecks): def frontend_session(wait_for_healthchecks):
session = requests.Session() session = requests.Session()
@ -25,15 +25,14 @@ def frontend_session(wait_for_healthchecks):
"Content-Type": "application/json", "Content-Type": "application/json",
} }
data = '{"username":"datahub", "password":"datahub"}' data = '{"username":"datahub", "password":"datahub"}'
response = session.post( response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data)
f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data
)
response.raise_for_status() response.raise_for_status()
yield session yield session
# TODO: Determine whether we need this or not. # TODO: Determine whether we need this or not.
@pytest.mark.dependency() @pytest.mark.dependency()
def test_healthchecks(wait_for_healthchecks): def test_healthchecks(wait_for_healthchecks):
# Call to wait_for_healthchecks fixture will do the actual functionality. # Call to wait_for_healthchecks fixture will do the actual functionality.
pass pass

81
smoke-test/tests/containers/containers_test.py
View File

@ -1,9 +1,6 @@
import pytest import pytest
import time from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest
from tests.utils import FRONTEND_ENDPOINT
from tests.utils import GMS_ENDPOINT
from tests.utils import ingest_file_via_rest
from tests.utils import delete_urns_from_file
@pytest.fixture(scope="module", autouse=False) @pytest.fixture(scope="module", autouse=False)
def ingest_cleanup_data(request): def ingest_cleanup_data(request):
@ -13,11 +10,13 @@ def ingest_cleanup_data(request):
print("removing containers test data") print("removing containers test data")
delete_urns_from_file("tests/containers/data.json") delete_urns_from_file("tests/containers/data.json")
@pytest.mark.dependency() @pytest.mark.dependency()
def test_healthchecks(wait_for_healthchecks): def test_healthchecks(wait_for_healthchecks):
# Call to wait_for_healthchecks fixture will do the actual functionality. # Call to wait_for_healthchecks fixture will do the actual functionality.
pass pass
@pytest.mark.dependency(depends=["test_healthchecks"]) @pytest.mark.dependency(depends=["test_healthchecks"])
def test_get_full_container(frontend_session, ingest_cleanup_data): def test_get_full_container(frontend_session, ingest_cleanup_data):
@ -96,13 +95,11 @@ def test_get_full_container(frontend_session, ingest_cleanup_data):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": container_urn},
"urn": container_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_container_json f"{get_frontend_url()}/api/v2/graphql", json=get_container_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -119,12 +116,15 @@ def test_get_full_container(frontend_session, ingest_cleanup_data):
assert container["properties"]["name"] == container_name assert container["properties"]["name"] == container_name
assert container["properties"]["description"] == container_description assert container["properties"]["description"] == container_description
assert container["subTypes"]["typeNames"][0] == "Schema" assert container["subTypes"]["typeNames"][0] == "Schema"
assert container["editableProperties"]["description"] == editable_container_description assert (
container["editableProperties"]["description"] == editable_container_description
)
assert container["ownership"] is None assert container["ownership"] is None
assert container["institutionalMemory"] is None assert container["institutionalMemory"] is None
assert container["tags"] is None assert container["tags"] is None
assert container["glossaryTerms"] is None assert container["glossaryTerms"] is None
@pytest.mark.dependency(depends=["test_healthchecks", "test_get_full_container"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_get_full_container"])
def test_get_parent_container(frontend_session, ingest_cleanup_data): def test_get_parent_container(frontend_session, ingest_cleanup_data):
@ -143,13 +143,11 @@ def test_get_parent_container(frontend_session, ingest_cleanup_data):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_dataset_json f"{get_frontend_url()}/api/v2/graphql", json=get_dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -162,6 +160,7 @@ def test_get_parent_container(frontend_session, ingest_cleanup_data):
dataset = res_data["data"]["dataset"] dataset = res_data["data"]["dataset"]
assert dataset["container"]["properties"]["name"] == "datahub_schema" assert dataset["container"]["properties"]["name"] == "datahub_schema"
@pytest.mark.dependency(depends=["test_healthchecks", "test_get_full_container"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_get_full_container"])
def test_update_container(frontend_session, ingest_cleanup_data): def test_update_container(frontend_session, ingest_cleanup_data):
@ -175,14 +174,14 @@ def test_update_container(frontend_session, ingest_cleanup_data):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"tagUrn": new_tag, "tagUrn": new_tag,
"resourceUrn": container_urn, "resourceUrn": container_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_tag_json f"{get_frontend_url()}/api/v2/graphql", json=add_tag_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -199,14 +198,14 @@ def test_update_container(frontend_session, ingest_cleanup_data):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"termUrn": new_term, "termUrn": new_term,
"resourceUrn": container_urn, "resourceUrn": container_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_term_json f"{get_frontend_url()}/api/v2/graphql", json=add_term_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -223,15 +222,15 @@ def test_update_container(frontend_session, ingest_cleanup_data):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"ownerUrn": new_owner, "ownerUrn": new_owner,
"resourceUrn": container_urn, "resourceUrn": container_urn,
"ownerEntityType": "CORP_USER" "ownerEntityType": "CORP_USER",
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_owner_json f"{get_frontend_url()}/api/v2/graphql", json=add_owner_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -248,15 +247,15 @@ def test_update_container(frontend_session, ingest_cleanup_data):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"linkUrl": new_link, "linkUrl": new_link,
"resourceUrn": container_urn, "resourceUrn": container_urn,
"label": "Label" "label": "Label",
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_link_json f"{get_frontend_url()}/api/v2/graphql", json=add_link_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -273,14 +272,14 @@ def test_update_container(frontend_session, ingest_cleanup_data):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"description": new_description, "description": new_description,
"resourceUrn": container_urn, "resourceUrn": container_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_description_json f"{get_frontend_url()}/api/v2/graphql", json=update_description_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -290,7 +289,7 @@ def test_update_container(frontend_session, ingest_cleanup_data):
assert res_data["data"]["updateDescription"] is True assert res_data["data"]["updateDescription"] is True
# Now fetch the container to ensure it was updated # Now fetch the container to ensure it was updated
# Get the container # Get the container
get_container_json = { get_container_json = {
"query": """query container($urn: String!) {\n "query": """query container($urn: String!) {\n
container(urn: $urn) {\n container(urn: $urn) {\n
@ -327,13 +326,11 @@ def test_update_container(frontend_session, ingest_cleanup_data):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": container_urn},
"urn": container_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_container_json f"{get_frontend_url()}/api/v2/graphql", json=get_container_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -348,4 +345,4 @@ def test_update_container(frontend_session, ingest_cleanup_data):
assert container["ownership"]["owners"][0]["owner"]["urn"] == new_owner assert container["ownership"]["owners"][0]["owner"]["urn"] == new_owner
assert container["institutionalMemory"]["elements"][0]["url"] == new_link assert container["institutionalMemory"]["elements"][0]["url"] == new_link
assert container["tags"]["tags"][0]["tag"]["urn"] == new_tag assert container["tags"]["tags"][0]["tag"]["urn"] == new_tag
assert container["glossaryTerms"]["terms"][0]["term"]["urn"] == new_term assert container["glossaryTerms"]["terms"][0]["term"]["urn"] == new_term

75
smoke-test/tests/deprecation/deprecation_test.py
View File

@ -1,9 +1,6 @@
import pytest import pytest
import time from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest
from tests.utils import FRONTEND_ENDPOINT
from tests.utils import GMS_ENDPOINT
from tests.utils import ingest_file_via_rest
from tests.utils import delete_urns_from_file
@pytest.fixture(scope="module", autouse=True) @pytest.fixture(scope="module", autouse=True)
def ingest_cleanup_data(request): def ingest_cleanup_data(request):
@ -13,14 +10,18 @@ def ingest_cleanup_data(request):
print("removing deprecation test data") print("removing deprecation test data")
delete_urns_from_file("tests/deprecation/data.json") delete_urns_from_file("tests/deprecation/data.json")
@pytest.mark.dependency() @pytest.mark.dependency()
def test_healthchecks(wait_for_healthchecks): def test_healthchecks(wait_for_healthchecks):
# Call to wait_for_healthchecks fixture will do the actual functionality. # Call to wait_for_healthchecks fixture will do the actual functionality.
pass pass
@pytest.mark.dependency(depends=["test_healthchecks"]) @pytest.mark.dependency(depends=["test_healthchecks"])
def test_update_deprecation_all_fields(frontend_session): def test_update_deprecation_all_fields(frontend_session):
dataset_urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" dataset_urn = (
"urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)"
)
dataset_json = { dataset_json = {
"query": """query getDataset($urn: String!) {\n "query": """query getDataset($urn: String!) {\n
@ -33,14 +34,12 @@ def test_update_deprecation_all_fields(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
# Fetch tags # Fetch tags
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -56,16 +55,16 @@ def test_update_deprecation_all_fields(frontend_session):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"urn": dataset_urn, "urn": dataset_urn,
"deprecated": True, "deprecated": True,
"note": "My test note", "note": "My test note",
"decommissionTime": 0 "decommissionTime": 0,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_deprecation_json f"{get_frontend_url()}/api/v2/graphql", json=update_deprecation_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -76,7 +75,7 @@ def test_update_deprecation_all_fields(frontend_session):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -85,30 +84,30 @@ def test_update_deprecation_all_fields(frontend_session):
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["deprecation"] == { assert res_data["data"]["dataset"]["deprecation"] == {
'deprecated': True, "deprecated": True,
'decommissionTime': 0, "decommissionTime": 0,
'note': 'My test note', "note": "My test note",
'actor': 'urn:li:corpuser:datahub' "actor": "urn:li:corpuser:datahub",
} }
@pytest.mark.dependency(depends=["test_healthchecks", "test_update_deprecation_all_fields"])
@pytest.mark.dependency(
depends=["test_healthchecks", "test_update_deprecation_all_fields"]
)
def test_update_deprecation_partial_fields(frontend_session, ingest_cleanup_data): def test_update_deprecation_partial_fields(frontend_session, ingest_cleanup_data):
dataset_urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" dataset_urn = (
"urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)"
)
update_deprecation_json = { update_deprecation_json = {
"query": """mutation updateDeprecation($input: UpdateDeprecationInput!) {\n "query": """mutation updateDeprecation($input: UpdateDeprecationInput!) {\n
updateDeprecation(input: $input) updateDeprecation(input: $input)
}""", }""",
"variables": { "variables": {"input": {"urn": dataset_urn, "deprecated": False}},
"input": {
"urn": dataset_urn,
"deprecated": False
}
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_deprecation_json f"{get_frontend_url()}/api/v2/graphql", json=update_deprecation_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -129,13 +128,11 @@ def test_update_deprecation_partial_fields(frontend_session, ingest_cleanup_data
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -144,8 +141,8 @@ def test_update_deprecation_partial_fields(frontend_session, ingest_cleanup_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["deprecation"] == { assert res_data["data"]["dataset"]["deprecation"] == {
'deprecated': False, "deprecated": False,
'note': '', "note": "",
'actor': 'urn:li:corpuser:datahub', "actor": "urn:li:corpuser:datahub",
'decommissionTime': None "decommissionTime": None,
} }

81
smoke-test/tests/domains/domains_test.py
View File

@ -1,9 +1,13 @@
import pytest
import time import time
from tests.utils import FRONTEND_ENDPOINT
from tests.utils import GMS_ENDPOINT import pytest
from tests.utils import ingest_file_via_rest from tests.utils import (
from tests.utils import delete_urns_from_file delete_urns_from_file,
get_frontend_url,
get_gms_url,
ingest_file_via_rest,
)
@pytest.fixture(scope="module", autouse=False) @pytest.fixture(scope="module", autouse=False)
def ingest_cleanup_data(request): def ingest_cleanup_data(request):
@ -13,11 +17,13 @@ def ingest_cleanup_data(request):
print("removing domains test data") print("removing domains test data")
delete_urns_from_file("tests/domains/data.json") delete_urns_from_file("tests/domains/data.json")
@pytest.mark.dependency() @pytest.mark.dependency()
def test_healthchecks(wait_for_healthchecks): def test_healthchecks(wait_for_healthchecks):
# Call to wait_for_healthchecks fixture will do the actual functionality. # Call to wait_for_healthchecks fixture will do the actual functionality.
pass pass
@pytest.mark.dependency(depends=["test_healthchecks"]) @pytest.mark.dependency(depends=["test_healthchecks"])
def test_create_list_get_domain(frontend_session): def test_create_list_get_domain(frontend_session):
@ -36,16 +42,11 @@ def test_create_list_get_domain(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"start": "0", "count": "20"}},
"input": {
"start": "0",
"count": "20"
}
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=list_domains_json f"{get_frontend_url()}/api/v2/graphql", json=list_domains_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -68,16 +69,16 @@ def test_create_list_get_domain(frontend_session):
createDomain(input: $input) createDomain(input: $input)
}""", }""",
"variables": { "variables": {
"input": { "input": {
"id": domain_id, "id": domain_id,
"name": domain_name, "name": domain_name,
"description": domain_description "description": domain_description,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=create_domain_json f"{get_frontend_url()}/api/v2/graphql", json=create_domain_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -94,7 +95,7 @@ def test_create_list_get_domain(frontend_session):
# Get new count of Domains # Get new count of Domains
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=list_domains_json f"{get_frontend_url()}/api/v2/graphql", json=list_domains_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -109,7 +110,6 @@ def test_create_list_get_domain(frontend_session):
print(after_count) print(after_count)
assert after_count == before_count + 1 assert after_count == before_count + 1
# Get the domain value back # Get the domain value back
get_domain_json = { get_domain_json = {
"query": """query domain($urn: String!) {\n "query": """query domain($urn: String!) {\n
@ -122,13 +122,11 @@ def test_create_list_get_domain(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": domain_urn},
"urn": domain_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_domain_json f"{get_frontend_url()}/api/v2/graphql", json=get_domain_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -139,18 +137,16 @@ def test_create_list_get_domain(frontend_session):
assert "errors" not in res_data assert "errors" not in res_data
domain = res_data["data"]["domain"] domain = res_data["data"]["domain"]
assert domain["urn"] == f'urn:li:domain:{domain_id}' assert domain["urn"] == f"urn:li:domain:{domain_id}"
assert domain["id"] == domain_id assert domain["id"] == domain_id
assert domain["properties"]["name"] == domain_name assert domain["properties"]["name"] == domain_name
assert domain["properties"]["description"] == domain_description assert domain["properties"]["description"] == domain_description
delete_json = { delete_json = {"urn": domain_urn}
"urn": domain_urn
}
# Cleanup: Delete the domain # Cleanup: Delete the domain
response = frontend_session.post( response = frontend_session.post(
f"{GMS_ENDPOINT}/entities?action=delete", json=delete_json f"{get_gms_url()}/entities?action=delete", json=delete_json
) )
response.raise_for_status() response.raise_for_status()
@ -160,20 +156,20 @@ def test_create_list_get_domain(frontend_session):
def test_set_unset_domain(frontend_session, ingest_cleanup_data): def test_set_unset_domain(frontend_session, ingest_cleanup_data):
# Set and Unset a Domain for a dataset. Note that this doesn't test for adding domains to charts, dashboards, charts, & jobs. # Set and Unset a Domain for a dataset. Note that this doesn't test for adding domains to charts, dashboards, charts, & jobs.
dataset_urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)" dataset_urn = (
"urn:li:dataset:(urn:li:dataPlatform:kafka,test-tags-terms-sample-kafka,PROD)"
)
domain_urn = "urn:li:domain:engineering" domain_urn = "urn:li:domain:engineering"
# First unset to be sure. # First unset to be sure.
unset_domain_json = { unset_domain_json = {
"query": """mutation unsetDomain($entityUrn: String!) {\n "query": """mutation unsetDomain($entityUrn: String!) {\n
unsetDomain(entityUrn: $entityUrn)}""", unsetDomain(entityUrn: $entityUrn)}""",
"variables": { "variables": {"entityUrn": dataset_urn},
"entityUrn": dataset_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=unset_domain_json f"{get_frontend_url()}/api/v2/graphql", json=unset_domain_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -187,14 +183,11 @@ def test_set_unset_domain(frontend_session, ingest_cleanup_data):
set_domain_json = { set_domain_json = {
"query": """mutation setDomain($entityUrn: String!, $domainUrn: String!) {\n "query": """mutation setDomain($entityUrn: String!, $domainUrn: String!) {\n
setDomain(entityUrn: $entityUrn, domainUrn: $domainUrn)}""", setDomain(entityUrn: $entityUrn, domainUrn: $domainUrn)}""",
"variables": { "variables": {"entityUrn": dataset_urn, "domainUrn": domain_urn},
"entityUrn": dataset_urn,
"domainUrn": domain_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=set_domain_json f"{get_frontend_url()}/api/v2/graphql", json=set_domain_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -204,7 +197,7 @@ def test_set_unset_domain(frontend_session, ingest_cleanup_data):
assert res_data["data"]["setDomain"] is True assert res_data["data"]["setDomain"] is True
assert "errors" not in res_data assert "errors" not in res_data
# Now, fetch the dataset's domain and confirm it was set.GMS_ENDPOINT # Now, fetch the dataset's domain and confirm it was set.
get_dataset_json = { get_dataset_json = {
"query": """query dataset($urn: String!) {\n "query": """query dataset($urn: String!) {\n
dataset(urn: $urn) {\n dataset(urn: $urn) {\n
@ -217,13 +210,11 @@ def test_set_unset_domain(frontend_session, ingest_cleanup_data):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_dataset_json f"{get_frontend_url()}/api/v2/graphql", json=get_dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()

237
smoke-test/tests/managed-ingestion/managed_ingestion_test.py
View File

@ -1,7 +1,8 @@
import pytest
import time import time
import requests
from tests.utils import FRONTEND_ENDPOINT import pytest
from tests.utils import get_frontend_url
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_create_list_get_remove_secret(frontend_session): def test_create_list_get_remove_secret(frontend_session):
@ -19,17 +20,10 @@ def test_create_list_get_remove_secret(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"start": "0", "count": "20"}},
"input": {
"start": "0",
"count": "20"
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -45,17 +39,10 @@ def test_create_list_get_remove_secret(frontend_session):
"query": """mutation createSecret($input: CreateSecretInput!) {\n "query": """mutation createSecret($input: CreateSecretInput!) {\n
createSecret(input: $input) createSecret(input: $input)
}""", }""",
"variables": { "variables": {"input": {"name": "SMOKE_TEST", "value": "mytestvalue"}},
"input": {
"name": "SMOKE_TEST",
"value": "mytestvalue"
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -82,17 +69,10 @@ def test_create_list_get_remove_secret(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"start": "0", "count": "20"}},
"input": {
"start": "0",
"count": "20"
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -113,16 +93,10 @@ def test_create_list_get_remove_secret(frontend_session):
value\n value\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"secrets": ["SMOKE_TEST"]}},
"input": {
"secrets": ["SMOKE_TEST"]
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -141,14 +115,10 @@ def test_create_list_get_remove_secret(frontend_session):
"query": """mutation deleteSecret($urn: String!) {\n "query": """mutation deleteSecret($urn: String!) {\n
deleteSecret(urn: $urn) deleteSecret(urn: $urn)
}""", }""",
"variables": { "variables": {"urn": secret_urn},
"urn": secret_urn
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -168,16 +138,10 @@ def test_create_list_get_remove_secret(frontend_session):
value\n value\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"secrets": ["SMOKE_TEST"]}},
"input": {
"secrets": ["SMOKE_TEST"]
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -190,6 +154,7 @@ def test_create_list_get_remove_secret(frontend_session):
secret_value_arr = [x for x in secret_values if x["name"] == "SMOKE_TEST"] secret_value_arr = [x for x in secret_values if x["name"] == "SMOKE_TEST"]
assert len(secret_value_arr) == 0 assert len(secret_value_arr) == 0
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_create_list_get_remove_ingestion_source(frontend_session): def test_create_list_get_remove_ingestion_source(frontend_session):
@ -205,17 +170,10 @@ def test_create_list_get_remove_ingestion_source(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"start": "0", "count": "20"}},
"input": {
"start": "0",
"count": "20"
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -232,26 +190,21 @@ def test_create_list_get_remove_ingestion_source(frontend_session):
createIngestionSource(input: $input) createIngestionSource(input: $input)
}""", }""",
"variables": { "variables": {
"input": { "input": {
"name": "My Test Ingestion Source", "name": "My Test Ingestion Source",
"type": "mysql", "type": "mysql",
"description": "My ingestion source description", "description": "My ingestion source description",
"schedule": { "schedule": {"interval": "* * * * *", "timezone": "UTC"},
"interval": "* * * * *", "config": {
"timezone": "UTC" "recipe": "MY_TEST_RECIPE",
}, "version": "0.8.18",
"config": { "executorId": "mytestexecutor",
"recipe": "MY_TEST_RECIPE", },
"version": "0.8.18", }
"executorId": "mytestexecutor" },
}
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -277,17 +230,10 @@ def test_create_list_get_remove_ingestion_source(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"start": "0", "count": "20"}},
"input": {
"start": "0",
"count": "20"
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -318,14 +264,10 @@ def test_create_list_get_remove_ingestion_source(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": ingestion_source_urn},
"urn": ingestion_source_urn
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -349,14 +291,10 @@ def test_create_list_get_remove_ingestion_source(frontend_session):
"query": """mutation deleteIngestionSource($urn: String!) {\n "query": """mutation deleteIngestionSource($urn: String!) {\n
deleteIngestionSource(urn: $urn) deleteIngestionSource(urn: $urn)
}""", }""",
"variables": { "variables": {"urn": ingestion_source_urn},
"urn": ingestion_source_urn
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -381,17 +319,10 @@ def test_create_list_get_remove_ingestion_source(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"start": "0", "count": "20"}},
"input": {
"start": "0",
"count": "20"
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -403,7 +334,14 @@ def test_create_list_get_remove_ingestion_source(frontend_session):
final_count = res_data["data"]["listIngestionSources"]["total"] final_count = res_data["data"]["listIngestionSources"]["total"]
assert final_count == after_count - 1 assert final_count == after_count - 1
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion", "test_create_list_get_remove_ingestion_source"])
@pytest.mark.dependency(
depends=[
"test_healthchecks",
"test_run_ingestion",
"test_create_list_get_remove_ingestion_source",
]
)
def test_create_list_get_ingestion_execution_request(frontend_session): def test_create_list_get_ingestion_execution_request(frontend_session):
# Create new ingestion source # Create new ingestion source
json = { json = {
@ -411,26 +349,21 @@ def test_create_list_get_ingestion_execution_request(frontend_session):
createIngestionSource(input: $input) createIngestionSource(input: $input)
}""", }""",
"variables": { "variables": {
"input": { "input": {
"name": "My Test Ingestion Source", "name": "My Test Ingestion Source",
"type": "mysql", "type": "mysql",
"description": "My ingestion source description", "description": "My ingestion source description",
"schedule": { "schedule": {"interval": "* * * * *", "timezone": "UTC"},
"interval": "* * * * *", "config": {
"timezone": "UTC" "recipe": "MY_TEST_RECIPE",
}, "version": "0.8.18",
"config": { "executorId": "mytestexecutor",
"recipe": "MY_TEST_RECIPE", },
"version": "0.8.18", }
"executorId": "mytestexecutor" },
}
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -446,16 +379,10 @@ def test_create_list_get_ingestion_execution_request(frontend_session):
"query": """mutation createIngestionExecutionRequest($input: CreateIngestionExecutionRequestInput!) {\n "query": """mutation createIngestionExecutionRequest($input: CreateIngestionExecutionRequestInput!) {\n
createIngestionExecutionRequest(input: $input) createIngestionExecutionRequest(input: $input)
}""", }""",
"variables": { "variables": {"input": {"ingestionSourceUrn": ingestion_source_urn}},
"input": {
"ingestionSourceUrn": ingestion_source_urn
}
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -483,14 +410,10 @@ def test_create_list_get_ingestion_execution_request(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": ingestion_source_urn},
"urn": ingestion_source_urn
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -501,7 +424,10 @@ def test_create_list_get_ingestion_execution_request(frontend_session):
ingestion_source = res_data["data"]["ingestionSource"] ingestion_source = res_data["data"]["ingestionSource"]
assert ingestion_source["executions"]["total"] == 1 assert ingestion_source["executions"]["total"] == 1
assert ingestion_source["executions"]["executionRequests"][0]["urn"] == execution_request_urn assert (
ingestion_source["executions"]["executionRequests"][0]["urn"]
== execution_request_urn
)
# Get the ingestion request back via direct lookup # Get the ingestion request back via direct lookup
json = { json = {
@ -522,14 +448,10 @@ def test_create_list_get_ingestion_execution_request(frontend_session):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": execution_request_urn},
"urn": execution_request_urn
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -544,10 +466,10 @@ def test_create_list_get_ingestion_execution_request(frontend_session):
# Verify input # Verify input
assert execution_request["input"]["task"] == "RUN_INGEST" assert execution_request["input"]["task"] == "RUN_INGEST"
assert len(execution_request["input"]["arguments"]) == 2 assert len(execution_request["input"]["arguments"]) == 2
assert execution_request["input"]["arguments"][0]["key"] == 'recipe' assert execution_request["input"]["arguments"][0]["key"] == "recipe"
assert execution_request["input"]["arguments"][0]["value"] == 'MY_TEST_RECIPE' assert execution_request["input"]["arguments"][0]["value"] == "MY_TEST_RECIPE"
assert execution_request["input"]["arguments"][1]["key"] == 'version' assert execution_request["input"]["arguments"][1]["key"] == "version"
assert execution_request["input"]["arguments"][1]["value"] == '0.8.18' assert execution_request["input"]["arguments"][1]["value"] == "0.8.18"
# Verify no result # Verify no result
assert execution_request["result"] is None assert execution_request["result"] is None
@ -557,14 +479,10 @@ def test_create_list_get_ingestion_execution_request(frontend_session):
"query": """mutation deleteIngestionSource($urn: String!) {\n "query": """mutation deleteIngestionSource($urn: String!) {\n
deleteIngestionSource(urn: $urn) deleteIngestionSource(urn: $urn)
}""", }""",
"variables": { "variables": {"urn": ingestion_source_urn},
"urn": ingestion_source_urn
}
} }
response = frontend_session.post( response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -572,4 +490,3 @@ def test_create_list_get_ingestion_execution_request(frontend_session):
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["deleteIngestionSource"] is not None assert res_data["data"]["deleteIngestionSource"] is not None
assert "errors" not in res_data assert "errors" not in res_data

292
smoke-test/tests/tags-and-terms/tags_and_terms_test.py
View File

@ -1,7 +1,6 @@
import pytest import pytest
from tests.utils import FRONTEND_ENDPOINT from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest
from tests.utils import ingest_file_via_rest
from tests.utils import delete_urns_from_file
@pytest.fixture(scope="module", autouse=True) @pytest.fixture(scope="module", autouse=True)
def ingest_cleanup_data(request): def ingest_cleanup_data(request):
@ -11,12 +10,11 @@ def ingest_cleanup_data(request):
print("removing test data") print("removing test data")
delete_urns_from_file("tests/tags-and-terms/data.json") delete_urns_from_file("tests/tags-and-terms/data.json")
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_add_tag(frontend_session,wait_for_healthchecks): def test_add_tag(frontend_session, wait_for_healthchecks):
platform = "urn:li:dataPlatform:kafka" platform = "urn:li:dataPlatform:kafka"
dataset_name = ( dataset_name = "test-tags-terms-sample-kafka"
"test-tags-terms-sample-kafka"
)
env = "PROD" env = "PROD"
dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})" dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
@ -34,14 +32,12 @@ def test_add_tag(frontend_session,wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
# Fetch tags # Fetch tags
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -57,14 +53,14 @@ def test_add_tag(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"tagUrn": "urn:li:tag:Legacy", "tagUrn": "urn:li:tag:Legacy",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json f"{get_frontend_url()}/api/v2/graphql", json=add_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -75,7 +71,7 @@ def test_add_tag(frontend_session,wait_for_healthchecks):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -83,7 +79,17 @@ def test_add_tag(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["globalTags"] == {'tags': [{'tag': {'description': 'Indicates the dataset is no longer supported', 'name': 'Legacy', 'urn': 'urn:li:tag:Legacy'}}]} assert res_data["data"]["dataset"]["globalTags"] == {
"tags": [
{
"tag": {
"description": "Indicates the dataset is no longer supported",
"name": "Legacy",
"urn": "urn:li:tag:Legacy",
}
}
]
}
remove_json = { remove_json = {
"query": """mutation removeTag($input: TagAssociationInput!) {\n "query": """mutation removeTag($input: TagAssociationInput!) {\n
@ -91,14 +97,14 @@ def test_add_tag(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"tagUrn": "urn:li:tag:Legacy", "tagUrn": "urn:li:tag:Legacy",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json f"{get_frontend_url()}/api/v2/graphql", json=remove_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -111,7 +117,7 @@ def test_add_tag(frontend_session,wait_for_healthchecks):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -119,10 +125,11 @@ def test_add_tag(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["globalTags"] == {'tags': [] } assert res_data["data"]["dataset"]["globalTags"] == {"tags": []}
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_add_tag_to_chart(frontend_session,wait_for_healthchecks): def test_add_tag_to_chart(frontend_session, wait_for_healthchecks):
chart_urn = "urn:li:chart:(looker,test-tags-terms-sample-chart)" chart_urn = "urn:li:chart:(looker,test-tags-terms-sample-chart)"
chart_json = { chart_json = {
@ -139,14 +146,12 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": chart_urn},
"urn": chart_urn
}
} }
# Fetch tags # Fetch tags
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=chart_json f"{get_frontend_url()}/api/v2/graphql", json=chart_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -162,14 +167,14 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"tagUrn": "urn:li:tag:Legacy", "tagUrn": "urn:li:tag:Legacy",
"resourceUrn": chart_urn, "resourceUrn": chart_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json f"{get_frontend_url()}/api/v2/graphql", json=add_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -180,7 +185,7 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=chart_json f"{get_frontend_url()}/api/v2/graphql", json=chart_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -188,7 +193,17 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["chart"] assert res_data["data"]["chart"]
assert res_data["data"]["chart"]["globalTags"] == {'tags': [{'tag': {'description': 'Indicates the dataset is no longer supported', 'name': 'Legacy', 'urn': 'urn:li:tag:Legacy'}}]} assert res_data["data"]["chart"]["globalTags"] == {
"tags": [
{
"tag": {
"description": "Indicates the dataset is no longer supported",
"name": "Legacy",
"urn": "urn:li:tag:Legacy",
}
}
]
}
remove_json = { remove_json = {
"query": """mutation removeTag($input: TagAssociationInput!) {\n "query": """mutation removeTag($input: TagAssociationInput!) {\n
@ -196,14 +211,14 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"tagUrn": "urn:li:tag:Legacy", "tagUrn": "urn:li:tag:Legacy",
"resourceUrn": chart_urn, "resourceUrn": chart_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json f"{get_frontend_url()}/api/v2/graphql", json=remove_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -214,7 +229,7 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=chart_json f"{get_frontend_url()}/api/v2/graphql", json=chart_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -222,14 +237,13 @@ def test_add_tag_to_chart(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["chart"] assert res_data["data"]["chart"]
assert res_data["data"]["chart"]["globalTags"] == {'tags': [] } assert res_data["data"]["chart"]["globalTags"] == {"tags": []}
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_add_term(frontend_session,wait_for_healthchecks): def test_add_term(frontend_session, wait_for_healthchecks):
platform = "urn:li:dataPlatform:kafka" platform = "urn:li:dataPlatform:kafka"
dataset_name = ( dataset_name = "test-tags-terms-sample-kafka"
"test-tags-terms-sample-kafka"
)
env = "PROD" env = "PROD"
dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})" dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
@ -246,15 +260,12 @@ def test_add_term(frontend_session,wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
# Fetch the terms # Fetch the terms
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -270,14 +281,14 @@ def test_add_term(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"termUrn": "urn:li:glossaryTerm:SavingAccount", "termUrn": "urn:li:glossaryTerm:SavingAccount",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json f"{get_frontend_url()}/api/v2/graphql", json=add_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -290,7 +301,7 @@ def test_add_term(frontend_session,wait_for_healthchecks):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -298,7 +309,16 @@ def test_add_term(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["glossaryTerms"] == {'terms': [{'term': {'name': 'SavingAccount', 'urn': 'urn:li:glossaryTerm:SavingAccount'}}]} assert res_data["data"]["dataset"]["glossaryTerms"] == {
"terms": [
{
"term": {
"name": "SavingAccount",
"urn": "urn:li:glossaryTerm:SavingAccount",
}
}
]
}
remove_json = { remove_json = {
"query": """mutation removeTerm($input: TermAssociationInput!) {\n "query": """mutation removeTerm($input: TermAssociationInput!) {\n
@ -306,14 +326,14 @@ def test_add_term(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"termUrn": "urn:li:glossaryTerm:SavingAccount", "termUrn": "urn:li:glossaryTerm:SavingAccount",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json f"{get_frontend_url()}/api/v2/graphql", json=remove_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -325,7 +345,7 @@ def test_add_term(frontend_session,wait_for_healthchecks):
assert res_data["data"]["removeTerm"] is True assert res_data["data"]["removeTerm"] is True
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_json f"{get_frontend_url()}/api/v2/graphql", json=dataset_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -333,14 +353,13 @@ def test_add_term(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["glossaryTerms"] == {'terms': []} assert res_data["data"]["dataset"]["glossaryTerms"] == {"terms": []}
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_update_schemafield(frontend_session,wait_for_healthchecks): def test_update_schemafield(frontend_session, wait_for_healthchecks):
platform = "urn:li:dataPlatform:kafka" platform = "urn:li:dataPlatform:kafka"
dataset_name = ( dataset_name = "test-tags-terms-sample-kafka"
"test-tags-terms-sample-kafka"
)
env = "PROD" env = "PROD"
dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})" dataset_urn = f"urn:li:dataset:({platform},{dataset_name},{env})"
@ -367,12 +386,10 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
dataset_schema_json_tags = { dataset_schema_json_tags = {
"query": """query getDataset($urn: String!) {\n "query": """query getDataset($urn: String!) {\n
dataset(urn: $urn) {\n dataset(urn: $urn) {\n
urn\n urn\n
@ -396,12 +413,10 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
dataset_schema_json_description = { dataset_schema_json_description = {
"query": """query getDataset($urn: String!) {\n "query": """query getDataset($urn: String!) {\n
dataset(urn: $urn) {\n dataset(urn: $urn) {\n
urn\n urn\n
@ -417,14 +432,12 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"urn": dataset_urn},
"urn": dataset_urn
}
} }
# dataset schema tags # dataset schema tags
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_tags f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_tags
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -440,16 +453,16 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"tagUrn": "urn:li:tag:Legacy", "tagUrn": "urn:li:tag:Legacy",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
"subResource": "[version=2.0].[type=boolean].field_bar", "subResource": "[version=2.0].[type=boolean].field_bar",
"subResourceType": "DATASET_FIELD" "subResourceType": "DATASET_FIELD",
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json f"{get_frontend_url()}/api/v2/graphql", json=add_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -460,7 +473,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
# Refetch the dataset schema # Refetch the dataset schema
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_tags f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_tags
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -468,7 +481,23 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'globalTags': {'tags': [{'tag': {'description': 'Indicates the dataset is no longer supported', 'name': 'Legacy', 'urn': 'urn:li:tag:Legacy'}}]}}]} assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {
"editableSchemaFieldInfo": [
{
"globalTags": {
"tags": [
{
"tag": {
"description": "Indicates the dataset is no longer supported",
"name": "Legacy",
"urn": "urn:li:tag:Legacy",
}
}
]
}
}
]
}
remove_json = { remove_json = {
"query": """mutation removeTag($input: TagAssociationInput!) {\n "query": """mutation removeTag($input: TagAssociationInput!) {\n
@ -476,16 +505,16 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"tagUrn": "urn:li:tag:Legacy", "tagUrn": "urn:li:tag:Legacy",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
"subResource": "[version=2.0].[type=boolean].field_bar", "subResource": "[version=2.0].[type=boolean].field_bar",
"subResourceType": "DATASET_FIELD" "subResourceType": "DATASET_FIELD",
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json f"{get_frontend_url()}/api/v2/graphql", json=remove_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -498,7 +527,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_tags f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_tags
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -506,7 +535,9 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'globalTags': {'tags': []}}]} assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {
"editableSchemaFieldInfo": [{"globalTags": {"tags": []}}]
}
add_json = { add_json = {
"query": """mutation addTerm($input: TermAssociationInput!) {\n "query": """mutation addTerm($input: TermAssociationInput!) {\n
@ -514,16 +545,16 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"termUrn": "urn:li:glossaryTerm:SavingAccount", "termUrn": "urn:li:glossaryTerm:SavingAccount",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
"subResource": "[version=2.0].[type=boolean].field_bar", "subResource": "[version=2.0].[type=boolean].field_bar",
"subResourceType": "DATASET_FIELD" "subResourceType": "DATASET_FIELD",
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=add_json f"{get_frontend_url()}/api/v2/graphql", json=add_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -534,7 +565,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
# Refetch the dataset schema # Refetch the dataset schema
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_terms f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_terms
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -542,7 +573,22 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'glossaryTerms': {'terms': [{'term': {'name': 'SavingAccount', 'urn': 'urn:li:glossaryTerm:SavingAccount'}}]}}]} assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {
"editableSchemaFieldInfo": [
{
"glossaryTerms": {
"terms": [
{
"term": {
"name": "SavingAccount",
"urn": "urn:li:glossaryTerm:SavingAccount",
}
}
]
}
}
]
}
remove_json = { remove_json = {
"query": """mutation removeTerm($input: TermAssociationInput!) {\n "query": """mutation removeTerm($input: TermAssociationInput!) {\n
@ -550,16 +596,16 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"termUrn": "urn:li:glossaryTerm:SavingAccount", "termUrn": "urn:li:glossaryTerm:SavingAccount",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
"subResource": "[version=2.0].[type=boolean].field_bar", "subResource": "[version=2.0].[type=boolean].field_bar",
"subResourceType": "DATASET_FIELD" "subResourceType": "DATASET_FIELD",
} }
} },
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=remove_json f"{get_frontend_url()}/api/v2/graphql", json=remove_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -570,7 +616,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_terms f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_terms
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -578,11 +624,13 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'glossaryTerms': {'terms': []}}]} assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {
"editableSchemaFieldInfo": [{"glossaryTerms": {"terms": []}}]
}
# dataset schema tags # dataset schema tags
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_tags f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_tags
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -593,17 +641,17 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
}""", }""",
"variables": { "variables": {
"input": { "input": {
"description": "new description", "description": "new description",
"resourceUrn": dataset_urn, "resourceUrn": dataset_urn,
"subResource": "[version=2.0].[type=boolean].field_bar", "subResource": "[version=2.0].[type=boolean].field_bar",
"subResourceType": "DATASET_FIELD" "subResourceType": "DATASET_FIELD",
} }
} },
} }
# fetch no description # fetch no description
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_description f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_description
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -611,10 +659,12 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{ 'description': None }]} assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {
"editableSchemaFieldInfo": [{"description": None}]
}
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_description_json f"{get_frontend_url()}/api/v2/graphql", json=update_description_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -625,7 +675,7 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
# Refetch the dataset # Refetch the dataset
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=dataset_schema_json_description f"{get_frontend_url()}/api/v2/graphql", json=dataset_schema_json_description
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -633,4 +683,6 @@ def test_update_schemafield(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {'editableSchemaFieldInfo': [{'description': 'new description'}]} assert res_data["data"]["dataset"]["editableSchemaMetadata"] == {
"editableSchemaFieldInfo": [{"description": "new description"}]
}

11
smoke-test/tests/test_stateful_ingestion.py
View File

@ -3,12 +3,13 @@ from typing import Any, Dict, Optional, cast
from datahub.ingestion.api.committable import StatefulCommittable from datahub.ingestion.api.committable import StatefulCommittable
from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.run.pipeline import Pipeline
from datahub.ingestion.source.sql.mysql import MySQLConfig, MySQLSource from datahub.ingestion.source.sql.mysql import MySQLConfig, MySQLSource
from datahub.ingestion.source.sql.sql_common import \ from datahub.ingestion.source.sql.sql_common import BaseSQLAlchemyCheckpointState
BaseSQLAlchemyCheckpointState
from datahub.ingestion.source.state.checkpoint import Checkpoint from datahub.ingestion.source.state.checkpoint import Checkpoint
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy.sql import text from sqlalchemy.sql import text
from tests.utils import get_gms_url
def test_stateful_ingestion(wait_for_healthchecks): def test_stateful_ingestion(wait_for_healthchecks):
def create_mysql_engine(mysql_source_config_dict: Dict[str, Any]) -> Any: def create_mysql_engine(mysql_source_config_dict: Dict[str, Any]) -> Any:
@ -57,7 +58,7 @@ def test_stateful_ingestion(wait_for_healthchecks):
"remove_stale_metadata": True, "remove_stale_metadata": True,
"state_provider": { "state_provider": {
"type": "datahub", "type": "datahub",
"config": {"datahub_api": {"server": "http://localhost:8080"}}, "config": {"datahub_api": {"server": get_gms_url()}},
}, },
}, },
} }
@ -69,13 +70,13 @@ def test_stateful_ingestion(wait_for_healthchecks):
}, },
"sink": { "sink": {
"type": "datahub-rest", "type": "datahub-rest",
"config": {"server": "http://localhost:8080"}, "config": {"server": get_gms_url()},
}, },
"pipeline_name": "mysql_stateful_ingestion_smoke_test_pipeline", "pipeline_name": "mysql_stateful_ingestion_smoke_test_pipeline",
"reporting": [ "reporting": [
{ {
"type": "datahub", "type": "datahub",
"config": {"datahub_api": {"server": "http://localhost:8080"}}, "config": {"datahub_api": {"server": get_gms_url()}},
} }
], ],
} }

168
smoke-test/tests/tests/tests_test.py
View File

@ -1,8 +1,7 @@
import pytest import pytest
import time
from tests.utils import FRONTEND_ENDPOINT from tests.utils import delete_urns_from_file, get_frontend_url, ingest_file_via_rest
from tests.utils import ingest_file_via_rest
from tests.utils import delete_urns_from_file
@pytest.fixture(scope="module", autouse=True) @pytest.fixture(scope="module", autouse=True)
def ingest_cleanup_data(request): def ingest_cleanup_data(request):
@ -12,17 +11,20 @@ def ingest_cleanup_data(request):
print("removing test data") print("removing test data")
delete_urns_from_file("tests/tests/data.json") delete_urns_from_file("tests/tests/data.json")
@pytest.mark.dependency() @pytest.mark.dependency()
def test_healthchecks(wait_for_healthchecks): def test_healthchecks(wait_for_healthchecks):
# Call to wait_for_healthchecks fixture will do the actual functionality. # Call to wait_for_healthchecks fixture will do the actual functionality.
pass pass
test_id = "test id" test_id = "test id"
test_name = "test name" test_name = "test name"
test_category = "test category" test_category = "test category"
test_description = "test description" test_description = "test description"
test_description = "test description" test_description = "test description"
def create_test(frontend_session): def create_test(frontend_session):
# Create new Test # Create new Test
@ -31,20 +33,18 @@ def create_test(frontend_session):
createTest(input: $input) createTest(input: $input)
}""", }""",
"variables": { "variables": {
"input": { "input": {
"id": test_id, "id": test_id,
"name": test_name, "name": test_name,
"category": test_category, "category": test_category,
"description": test_description, "description": test_description,
"definition": { "definition": {"json": "{}"},
"json": "{}" }
} },
}
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=create_test_json f"{get_frontend_url()}/api/v2/graphql", json=create_test_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -56,23 +56,23 @@ def create_test(frontend_session):
return res_data["data"]["createTest"] return res_data["data"]["createTest"]
def delete_test(frontend_session, test_urn): def delete_test(frontend_session, test_urn):
delete_test_json = { delete_test_json = {
"query": """mutation deleteTest($urn: String!) {\n "query": """mutation deleteTest($urn: String!) {\n
deleteTest(urn: $urn) deleteTest(urn: $urn)
}""", }""",
"variables": { "variables": {"urn": test_urn},
"urn": test_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=delete_test_json f"{get_frontend_url()}/api/v2/graphql", json=delete_test_json
) )
response.raise_for_status() response.raise_for_status()
@pytest.mark.dependency(depends=["test_healthchecks"]) @pytest.mark.dependency(depends=["test_healthchecks"])
def test_create_test(frontend_session,wait_for_healthchecks): def test_create_test(frontend_session, wait_for_healthchecks):
test_urn = create_test(frontend_session) test_urn = create_test(frontend_session)
@ -89,12 +89,10 @@ def test_create_test(frontend_session,wait_for_healthchecks):
}\n }\n
} }
}""", }""",
"variables": { "variables": {"urn": test_urn},
"urn": test_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_test_json f"{get_frontend_url()}/api/v2/graphql", json=get_test_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -102,13 +100,13 @@ def test_create_test(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["test"] == { assert res_data["data"]["test"] == {
"urn": test_urn, "urn": test_urn,
"name": test_name, "name": test_name,
"category": test_category, "category": test_category,
"description": test_description, "description": test_description,
"definition": { "definition": {
"json": "{}", "json": "{}",
} },
} }
assert "errors" not in res_data assert "errors" not in res_data
@ -117,7 +115,7 @@ def test_create_test(frontend_session,wait_for_healthchecks):
# Ensure the test no longer exists # Ensure the test no longer exists
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_test_json f"{get_frontend_url()}/api/v2/graphql", json=get_test_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -127,7 +125,7 @@ def test_create_test(frontend_session,wait_for_healthchecks):
@pytest.mark.dependency(depends=["test_healthchecks", "test_create_test"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_create_test"])
def test_update_test(frontend_session,wait_for_healthchecks): def test_update_test(frontend_session, wait_for_healthchecks):
test_urn = create_test(frontend_session) test_urn = create_test(frontend_session)
test_name = "new name" test_name = "new name"
test_category = "new category" test_category = "new category"
@ -140,20 +138,18 @@ def test_update_test(frontend_session,wait_for_healthchecks):
updateTest(urn: $urn, input: $input) updateTest(urn: $urn, input: $input)
}""", }""",
"variables": { "variables": {
"urn": test_urn, "urn": test_urn,
"input": { "input": {
"name": test_name, "name": test_name,
"category": test_category, "category": test_category,
"description": test_description, "description": test_description,
"definition": { "definition": {"json": "{}"},
"json": "{}" },
} },
}
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=update_test_json f"{get_frontend_url()}/api/v2/graphql", json=update_test_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -176,12 +172,10 @@ def test_update_test(frontend_session,wait_for_healthchecks):
}\n }\n
} }
}""", }""",
"variables": { "variables": {"urn": test_urn},
"urn": test_urn
}
} }
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=get_test_json f"{get_frontend_url()}/api/v2/graphql", json=get_test_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -189,22 +183,23 @@ def test_update_test(frontend_session,wait_for_healthchecks):
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["test"] == { assert res_data["data"]["test"] == {
"urn": test_urn, "urn": test_urn,
"name": test_name, "name": test_name,
"category": test_category, "category": test_category,
"description": test_description, "description": test_description,
"definition": { "definition": {
"json": "{}", "json": "{}",
} },
} }
assert "errors" not in res_data assert "errors" not in res_data
delete_test(frontend_session, test_urn) delete_test(frontend_session, test_urn)
@pytest.mark.dependency(depends=["test_healthchecks", "test_update_test"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_update_test"])
def test_list_tests(frontend_session,wait_for_healthchecks): def test_list_tests(frontend_session, wait_for_healthchecks):
list_tests_json = { list_tests_json = {
"query": """query listTests($input: ListTestsInput!) {\n "query": """query listTests($input: ListTestsInput!) {\n
listTests(input: $input) {\n listTests(input: $input) {\n
start\n start\n
count\n count\n
@ -214,30 +209,27 @@ def test_list_tests(frontend_session,wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": {"start": "0", "count": "20"}},
"input": { }
"start": "0",
"count": "20"
}
}
}
response = frontend_session.post( response = frontend_session.post(
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=list_tests_json f"{get_frontend_url()}/api/v2/graphql", json=list_tests_json
) )
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
assert res_data assert res_data
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["listTests"]["total"] >= 2 assert res_data["data"]["listTests"]["total"] >= 2
assert len(res_data["data"]["listTests"]["tests"]) >= 2 assert len(res_data["data"]["listTests"]["tests"]) >= 2
assert "errors" not in res_data assert "errors" not in res_data
@pytest.mark.dependency(depends=["test_healthchecks"]) @pytest.mark.dependency(depends=["test_healthchecks"])
def test_get_test_results(frontend_session,wait_for_healthchecks): def test_get_test_results(frontend_session, wait_for_healthchecks):
urn = "urn:li:dataset:(urn:li:dataPlatform:kafka,test-tests-sample,PROD)" # Test urn urn = (
"urn:li:dataset:(urn:li:dataPlatform:kafka,test-tests-sample,PROD)" # Test urn
)
json = { json = {
"query": """query getDataset($urn: String!) {\n "query": """query getDataset($urn: String!) {\n
dataset(urn: $urn) {\n dataset(urn: $urn) {\n
@ -258,9 +250,9 @@ def test_get_test_results(frontend_session,wait_for_healthchecks):
}\n }\n
}\n }\n
}""", }""",
"variables": {"urn": urn }, "variables": {"urn": urn},
} }
response = frontend_session.post(f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json) response = frontend_session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
response.raise_for_status() response.raise_for_status()
res_data = response.json() res_data = response.json()
@ -269,20 +261,6 @@ def test_get_test_results(frontend_session,wait_for_healthchecks):
assert res_data["data"]["dataset"] assert res_data["data"]["dataset"]
assert res_data["data"]["dataset"]["urn"] == urn assert res_data["data"]["dataset"]["urn"] == urn
assert res_data["data"]["dataset"]["testResults"] == { assert res_data["data"]["dataset"]["testResults"] == {
"failing": [ "failing": [{"test": {"urn": "urn:li:test:test-1"}, "type": "FAILURE"}],
{ "passing": [{"test": {"urn": "urn:li:test:test-2"}, "type": "SUCCESS"}],
"test": {
"urn": "urn:li:test:test-1"
},
"type": "FAILURE"
}
],
"passing": [
{
"test": {
"urn": "urn:li:test:test-2"
},
"type": "SUCCESS"
}
]
} }

117
smoke-test/tests/tokens/revokable_access_token_test.py
View File

@ -1,10 +1,9 @@
import pytest
import time
import requests
from tests.utils import FRONTEND_ENDPOINT
from time import sleep from time import sleep
from tests.utils import ingest_file_via_rest
from datahub.cli.ingest_cli import get_session_and_host import pytest
import requests
from tests.utils import get_frontend_url, ingest_file_via_rest
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def test_setup(): def test_setup():
@ -28,7 +27,8 @@ def test_setup():
# Clean up # Clean up
res_data = listAccessTokens(admin_session) res_data = listAccessTokens(admin_session)
for metadata in res_data["data"]["listAccessTokens"]["tokens"]: for metadata in res_data["data"]["listAccessTokens"]["tokens"]:
revokeAccessToken(admin_session, metadata["id"]) revokeAccessToken(admin_session, metadata["id"])
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_admin_can_create_list_and_revoke_tokens(): def test_admin_can_create_list_and_revoke_tokens():
@ -47,7 +47,10 @@ def test_admin_can_create_list_and_revoke_tokens():
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["createAccessToken"] assert res_data["data"]["createAccessToken"]
assert res_data["data"]["createAccessToken"]["accessToken"] assert res_data["data"]["createAccessToken"]["accessToken"]
assert res_data["data"]["createAccessToken"]["metadata"]["actorUrn"] == "urn:li:corpuser:datahub" assert (
res_data["data"]["createAccessToken"]["metadata"]["actorUrn"]
== "urn:li:corpuser:datahub"
)
admin_tokenId = res_data["data"]["createAccessToken"]["metadata"]["id"] admin_tokenId = res_data["data"]["createAccessToken"]["metadata"]["id"]
# Using a super account, list the previously created token. # Using a super account, list the previously created token.
@ -56,8 +59,14 @@ def test_admin_can_create_list_and_revoke_tokens():
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["listAccessTokens"]["total"] is not None assert res_data["data"]["listAccessTokens"]["total"] is not None
assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 1 assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 1
assert res_data["data"]["listAccessTokens"]["tokens"][1]["actorUrn"] == "urn:li:corpuser:datahub" assert (
assert res_data["data"]["listAccessTokens"]["tokens"][1]["ownerUrn"] == "urn:li:corpuser:datahub" res_data["data"]["listAccessTokens"]["tokens"][1]["actorUrn"]
== "urn:li:corpuser:datahub"
)
assert (
res_data["data"]["listAccessTokens"]["tokens"][1]["ownerUrn"]
== "urn:li:corpuser:datahub"
)
# Check that the super account can revoke tokens that it created # Check that the super account can revoke tokens that it created
res_data = revokeAccessToken(admin_session, admin_tokenId) res_data = revokeAccessToken(admin_session, admin_tokenId)
@ -73,6 +82,7 @@ def test_admin_can_create_list_and_revoke_tokens():
assert res_data["data"]["listAccessTokens"]["total"] is not None assert res_data["data"]["listAccessTokens"]["total"] is not None
assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_admin_can_create_and_revoke_tokens_for_other_user(): def test_admin_can_create_and_revoke_tokens_for_other_user():
admin_session = loginAs("datahub", "datahub") admin_session = loginAs("datahub", "datahub")
@ -90,7 +100,10 @@ def test_admin_can_create_and_revoke_tokens_for_other_user():
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["createAccessToken"] assert res_data["data"]["createAccessToken"]
assert res_data["data"]["createAccessToken"]["accessToken"] assert res_data["data"]["createAccessToken"]["accessToken"]
assert res_data["data"]["createAccessToken"]["metadata"]["actorUrn"] == "urn:li:corpuser:user" assert (
res_data["data"]["createAccessToken"]["metadata"]["actorUrn"]
== "urn:li:corpuser:user"
)
user_tokenId = res_data["data"]["createAccessToken"]["metadata"]["id"] user_tokenId = res_data["data"]["createAccessToken"]["metadata"]["id"]
# Using a super account, list the previously created tokens. # Using a super account, list the previously created tokens.
@ -99,8 +112,14 @@ def test_admin_can_create_and_revoke_tokens_for_other_user():
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["listAccessTokens"]["total"] is not None assert res_data["data"]["listAccessTokens"]["total"] is not None
assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 1 assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 1
assert res_data["data"]["listAccessTokens"]["tokens"][0]["actorUrn"] == "urn:li:corpuser:user" assert (
assert res_data["data"]["listAccessTokens"]["tokens"][0]["ownerUrn"] == "urn:li:corpuser:datahub" res_data["data"]["listAccessTokens"]["tokens"][0]["actorUrn"]
== "urn:li:corpuser:user"
)
assert (
res_data["data"]["listAccessTokens"]["tokens"][0]["ownerUrn"]
== "urn:li:corpuser:datahub"
)
# Check that the super account can revoke tokens that it created for another user # Check that the super account can revoke tokens that it created for another user
res_data = revokeAccessToken(admin_session, user_tokenId) res_data = revokeAccessToken(admin_session, user_tokenId)
@ -115,6 +134,8 @@ def test_admin_can_create_and_revoke_tokens_for_other_user():
assert res_data["data"] assert res_data["data"]
assert res_data["data"]["listAccessTokens"]["total"] is not None assert res_data["data"]["listAccessTokens"]["total"] is not None
assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0 assert len(res_data["data"]["listAccessTokens"]["tokens"]) == 0
""" """
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
def test_non_admin_can_create_list_revoke_tokens(): def test_non_admin_can_create_list_revoke_tokens():
@ -215,8 +236,10 @@ def test_non_admin_can_not_generate_tokens_for_others():
assert res_data["errors"] assert res_data["errors"]
assert res_data["errors"][0]["message"] == "Unauthorized to perform this action. Please contact your DataHub administrator." assert res_data["errors"][0]["message"] == "Unauthorized to perform this action. Please contact your DataHub administrator."
""" """
def generateAccessToken_v1(session, actorUrn): def generateAccessToken_v1(session, actorUrn):
# Create new token # Create new token
json = { json = {
"query": """query getAccessToken($input: GetAccessTokenInput!) {\n "query": """query getAccessToken($input: GetAccessTokenInput!) {\n
getAccessToken(input: $input) {\n getAccessToken(input: $input) {\n
@ -224,22 +247,17 @@ def generateAccessToken_v1(session, actorUrn):
}\n }\n
}""", }""",
"variables": { "variables": {
"input": { "input": {"type": "PERSONAL", "actorUrn": actorUrn, "duration": "ONE_HOUR"}
"type": "PERSONAL", },
"actorUrn": actorUrn,
"duration": "ONE_HOUR"
}
}
} }
response = session.post( response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
def generateAccessToken_v2(session, actorUrn): def generateAccessToken_v2(session, actorUrn):
# Create new token # Create new token
json = { json = {
"query": """mutation createAccessToken($input: CreateAccessTokenInput!) {\n "query": """mutation createAccessToken($input: CreateAccessTokenInput!) {\n
createAccessToken(input: $input) {\n createAccessToken(input: $input) {\n
@ -254,32 +272,31 @@ def generateAccessToken_v2(session, actorUrn):
}\n }\n
}""", }""",
"variables": { "variables": {
"input": { "input": {
"type": "PERSONAL", "type": "PERSONAL",
"actorUrn": actorUrn, "actorUrn": actorUrn,
"duration": "ONE_HOUR", "duration": "ONE_HOUR",
"name": "my token" "name": "my token",
} }
} },
} }
response = session.post( response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
sleep(5) sleep(5)
return response.json() return response.json()
def listAccessTokens(session, filters=[]): def listAccessTokens(session, filters=[]):
# Get count of existing tokens # Get count of existing tokens
input = { input = {
"start": "0", "start": "0",
"count": "20", "count": "20",
} }
if filters: if filters:
input['filters'] = filters input["filters"] = filters
json = { json = {
"query": """query listAccessTokens($input: ListAccessTokenInput!) {\n "query": """query listAccessTokens($input: ListAccessTokenInput!) {\n
@ -295,45 +312,37 @@ def listAccessTokens(session, filters=[]):
}\n }\n
}\n }\n
}""", }""",
"variables": { "variables": {"input": input},
"input": input
}
} }
response = session.post( response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
def revokeAccessToken(session, tokenId): def revokeAccessToken(session, tokenId):
# Revoke token # Revoke token
json = { json = {
"query": """mutation revokeAccessToken($tokenId: String!) {\n "query": """mutation revokeAccessToken($tokenId: String!) {\n
revokeAccessToken(tokenId: $tokenId) revokeAccessToken(tokenId: $tokenId)
}""", }""",
"variables": { "variables": {"tokenId": tokenId},
"tokenId": tokenId
}
} }
response = session.post( response = session.post(f"{get_frontend_url()}/api/v2/graphql", json=json)
f"{FRONTEND_ENDPOINT}/api/v2/graphql", json=json
)
sleep(5) sleep(5)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
def loginAs(username, password): def loginAs(username, password):
session = requests.Session() session = requests.Session()
headers = { headers = {
"Content-Type": "application/json", "Content-Type": "application/json",
} }
data = '{"username":"' + username +'", "password":"' + password + '"}' data = '{"username":"' + username + '", "password":"' + password + '"}'
response = session.post( response = session.post(f"{get_frontend_url()}/logIn", headers=headers, data=data)
f"{FRONTEND_ENDPOINT}/logIn", headers=headers, data=data
)
response.raise_for_status() response.raise_for_status()
return session return session

38
smoke-test/tests/utils.py
View File

@ -1,12 +1,30 @@
import json import json
import os
from typing import Any
import requests import requests
from typing import Any
from datahub.cli import cli_utils from datahub.cli import cli_utils
from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.run.pipeline import Pipeline
GMS_ENDPOINT = "http://localhost:8080"
FRONTEND_ENDPOINT = "http://localhost:9002" def get_gms_url():
return os.getenv("DATAHUB_GMS_URL") or "http://localhost:8080"
def get_frontend_url():
return os.getenv("DATAHUB_FRONTEND_URL") or "http://localhost:9002"
def get_kafka_broker_url():
return os.getenv("DATAHUB_KAFKA_URL") or "localhost:9092"
def get_sleep_info():
return (
os.environ.get("DATAHUB_TEST_SLEEP_BETWEEN") or 60,
os.environ.get("DATAHUB_TEST_SLEEP_TIMES") or 5,
)
def ingest_file_via_rest(filename: str) -> Any: def ingest_file_via_rest(filename: str) -> Any:
pipeline = Pipeline.create( pipeline = Pipeline.create(
@ -17,7 +35,7 @@ def ingest_file_via_rest(filename: str) -> Any:
}, },
"sink": { "sink": {
"type": "datahub-rest", "type": "datahub-rest",
"config": {"server": GMS_ENDPOINT}, "config": {"server": get_gms_url()},
}, },
} }
) )
@ -39,19 +57,19 @@ def delete_urns_from_file(filename: str) -> None:
with open(filename) as f: with open(filename) as f:
d = json.load(f) d = json.load(f)
for entry in d: for entry in d:
is_mcp = 'entityUrn' in entry is_mcp = "entityUrn" in entry
urn = None urn = None
# Kill Snapshot # Kill Snapshot
if is_mcp: if is_mcp:
urn = entry['entityUrn'] urn = entry["entityUrn"]
else: else:
snapshot_union = entry['proposedSnapshot'] snapshot_union = entry["proposedSnapshot"]
snapshot = list(snapshot_union.values())[0] snapshot = list(snapshot_union.values())[0]
urn = snapshot['urn'] urn = snapshot["urn"]
payload_obj = {"urn": urn} payload_obj = {"urn": urn}
cli_utils.post_delete_endpoint_with_session_and_url( cli_utils.post_delete_endpoint_with_session_and_url(
session, session,
GMS_ENDPOINT + "/entities?action=delete", get_gms_url() + "/entities?action=delete",
payload_obj, payload_obj,
) )