yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-03 22:23:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingestion): Fix looker test (#2601)

Browse Source
This commit is contained in:
Fredrik Sannholm 2021-05-25 21:15:47 +03:00 committed by GitHub
parent 67e1512636
commit 1e0b67ce56
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 108 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
metadata-ingestion/tests/integration/looker/expected_output.json
View File

@ -1,4 +1,41 @@
[
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {
"urn": "urn:li:chart:(looker,dashboard_elements.2)",
"aspects": [
{
"com.linkedin.pegasus2avro.chart.ChartInfo": {
"customProperties": {},
"externalUrl": null,
"title": "",
"description": "Some text",
"lastModified": {
"created": {
"time": 1615443388000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"lastModified": {
"time": 1615443388000,
"actor": "urn:li:corpuser:etl",
"impersonator": null
},
"deleted": null
},
"chartUrl": "https://looker.company.com/x/",
"inputs": [],
"type": null,
"access": null,
"lastRefreshed": null
}
}
]
}
},
"proposedDelta": null
},
{
"auditHeader": null,
"proposedSnapshot": {
@ -11,7 +48,9 @@
"externalUrl": null,
"title": "foo",
"description": "lorem ipsum",
"charts": [],
"charts": [
"urn:li:chart:(looker,dashboard_elements.2)"
],
"lastModified": {
"created": {
"time": 1615443388000,

70
metadata-ingestion/tests/integration/looker/test_looker.py
View File

@ -28,7 +28,7 @@ def test_looker_ingest(pytestconfig, tmp_path, mock_time):
query=Query(
model="data",
view="my_view",
dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}',
dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}]',
),
)
],
@ -57,7 +57,73 @@ def test_looker_ingest(pytestconfig, tmp_path, mock_time):
)
pipeline.run()
pipeline.raise_from_status()
output = mce_helpers.load_json_file(str(tmp_path / "looker_mces.json"))
expected = mce_helpers.load_json_file(
str(test_resources_dir / "expected_output.json")
)
mce_helpers.assert_mces_equal(output, expected)
def test_looker_ingest_allow_pattern(pytestconfig, tmp_path, mock_time):
mocked_client = mock.MagicMock()
with mock.patch(
"datahub.ingestion.source.looker.LookerDashboardSource._get_looker_client",
mocked_client,
):
mocked_client.return_value.all_dashboards.return_value = [Dashboard(id="1")]
mocked_client.return_value.dashboard.return_value = Dashboard(
id="1",
title="foo",
created_at=datetime.utcfromtimestamp(time.time()),
description="lorem ipsum",
dashboard_elements=[
DashboardElement(
id="2",
type="",
subtitle_text="Some text",
query=Query(
model="data",
view="my_view",
dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}]',
),
),
DashboardElement(
id="10",
type="",
subtitle_text="Some other text",
query=Query(
model="bogus data",
view="my_view",
dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}]',
),
),
],
)
test_resources_dir = pytestconfig.rootpath / "tests/integration/looker"
pipeline = Pipeline.create(
{
"run_id": "looker-test",
"source": {
"type": "looker",
"config": {
"base_url": "https://looker.company.com",
"client_id": "foo",
"client_secret": "bar",
"chart_pattern": {"allow": ["2"]},
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/looker_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
output = mce_helpers.load_json_file(str(tmp_path / "looker_mces.json"))
expected = mce_helpers.load_json_file(
str(test_resources_dir / "expected_output.json")