diff --git a/metadata-ingestion/tests/integration/looker/expected_output.json b/metadata-ingestion/tests/integration/looker/expected_output.json index 16203bee47..84e759d5d4 100644 --- a/metadata-ingestion/tests/integration/looker/expected_output.json +++ b/metadata-ingestion/tests/integration/looker/expected_output.json @@ -1,4 +1,41 @@ [ + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,dashboard_elements.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": null, + "title": "", + "description": "Some text", + "lastModified": { + "created": { + "time": 1615443388000, + "actor": "urn:li:corpuser:etl", + "impersonator": null + }, + "lastModified": { + "time": 1615443388000, + "actor": "urn:li:corpuser:etl", + "impersonator": null + }, + "deleted": null + }, + "chartUrl": "https://looker.company.com/x/", + "inputs": [], + "type": null, + "access": null, + "lastRefreshed": null + } + } + ] + } + }, + "proposedDelta": null + }, { "auditHeader": null, "proposedSnapshot": { @@ -11,7 +48,9 @@ "externalUrl": null, "title": "foo", "description": "lorem ipsum", - "charts": [], + "charts": [ + "urn:li:chart:(looker,dashboard_elements.2)" + ], "lastModified": { "created": { "time": 1615443388000, diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py index 93ad9a1b51..ab95b3a8db 100644 --- a/metadata-ingestion/tests/integration/looker/test_looker.py +++ b/metadata-ingestion/tests/integration/looker/test_looker.py @@ -28,7 +28,7 @@ def test_looker_ingest(pytestconfig, tmp_path, mock_time): query=Query( model="data", view="my_view", - dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}', + dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}]', ), ) ], @@ -57,7 +57,73 @@ def test_looker_ingest(pytestconfig, tmp_path, mock_time): ) pipeline.run() pipeline.raise_from_status() - + output = mce_helpers.load_json_file(str(tmp_path / "looker_mces.json")) + expected = mce_helpers.load_json_file( + str(test_resources_dir / "expected_output.json") + ) + mce_helpers.assert_mces_equal(output, expected) + + +def test_looker_ingest_allow_pattern(pytestconfig, tmp_path, mock_time): + mocked_client = mock.MagicMock() + with mock.patch( + "datahub.ingestion.source.looker.LookerDashboardSource._get_looker_client", + mocked_client, + ): + mocked_client.return_value.all_dashboards.return_value = [Dashboard(id="1")] + mocked_client.return_value.dashboard.return_value = Dashboard( + id="1", + title="foo", + created_at=datetime.utcfromtimestamp(time.time()), + description="lorem ipsum", + dashboard_elements=[ + DashboardElement( + id="2", + type="", + subtitle_text="Some text", + query=Query( + model="data", + view="my_view", + dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}]', + ), + ), + DashboardElement( + id="10", + type="", + subtitle_text="Some other text", + query=Query( + model="bogus data", + view="my_view", + dynamic_fields='[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}]', + ), + ), + ], + ) + + test_resources_dir = pytestconfig.rootpath / "tests/integration/looker" + + pipeline = Pipeline.create( + { + "run_id": "looker-test", + "source": { + "type": "looker", + "config": { + "base_url": "https://looker.company.com", + "client_id": "foo", + "client_secret": "bar", + "chart_pattern": {"allow": ["2"]}, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/looker_mces.json", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() output = mce_helpers.load_json_file(str(tmp_path / "looker_mces.json")) expected = mce_helpers.load_json_file( str(test_resources_dir / "expected_output.json")