diff --git a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py index 1201ec8426e..0e422908388 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py @@ -79,6 +79,8 @@ class DomodashboardSource(DashboardServiceSource): return cls(config, metadata) def get_dashboards_list(self) -> Optional[List[DomoDashboardDetails]]: + if not self.source_config.includeOwners: + logger.debug("Skipping owner information as includeOwners is False") dashboards = self.client.domo.page_list() dashboard_list = [] for dashboard in dashboards: @@ -104,15 +106,23 @@ class DomodashboardSource(DashboardServiceSource): def get_owner_ref( self, dashboard_details: DomoDashboardDetails ) -> Optional[EntityReferenceList]: - for owner in dashboard_details.owners or []: - try: - owner_details = self.client.domo.users_get(owner.id) - if owner_details.get("email"): - return self.metadata.get_reference_by_email(owner_details["email"]) - except Exception as exc: - logger.warning( - f"Error while getting details of user {owner.displayName} - {exc}" - ) + try: + if not self.source_config.includeOwners: + return None + for owner in dashboard_details.owners or []: + try: + owner_details = self.client.domo.users_get(owner.id) + if owner_details.get("email"): + return self.metadata.get_reference_by_email( + owner_details["email"] + ) + except Exception as exc: + logger.warning( + f"Error while getting details of user {owner.displayName} - {exc}" + ) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.warning(f"Could not fetch owner data due to {err}") return None def yield_dashboard( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index 940242bf8b2..3fabb6c6c99 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -922,6 +922,8 @@ class LookerSource(DashboardServiceSource): """ Get List of all dashboards """ + if not self.source_config.includeOwners: + logger.debug("Skipping owner information as includeOwners is False") try: return list( self.client.all_dashboards(fields=",".join(LIST_DASHBOARD_FIELDS)) @@ -963,6 +965,8 @@ class LookerSource(DashboardServiceSource): Optional[EntityReference] """ try: + if not self.source_config.includeOwners: + return None if dashboard_details.user_id is not None: dashboard_owner = self.client.user(dashboard_details.user_id) if dashboard_owner.email: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py index 3f3c30299a8..3b358887a1e 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py @@ -108,6 +108,8 @@ class MetabaseSource(DashboardServiceSource): """ Get List of all dashboards """ + if not self.source_config.includeOwners: + logger.debug("Skipping owner information as includeOwners is False") self.dashboards_list = self.client.get_dashboards_list(self.collections) return self.dashboards_list @@ -180,6 +182,8 @@ class MetabaseSource(DashboardServiceSource): Get dashboard owner from email """ try: + if not self.source_config.includeOwners: + return None if dashboard_details.creator_id: owner_details = self.client.get_user_details( dashboard_details.creator_id diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py index 710edbb199e..b02a3856667 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py @@ -110,6 +110,8 @@ class RedashSource(DashboardServiceSource): ) def get_dashboards_list(self) -> Optional[List[dict]]: + if not self.source_config.includeOwners: + logger.debug("Skipping owner information as includeOwners is False") return self.dashboard_list def get_dashboard_name(self, dashboard: dict) -> str: @@ -123,6 +125,8 @@ class RedashSource(DashboardServiceSource): Get owner from email """ try: + if not self.source_config.includeOwners: + return None if dashboard_details.get("user") and dashboard_details["user"].get("email"): return self.metadata.get_reference_by_email( dashboard_details["user"].get("email") diff --git a/ingestion/src/metadata/ingestion/source/dashboard/sigma/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/sigma/metadata.py index d8e89cb242b..38a62e8f1d3 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/sigma/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/sigma/metadata.py @@ -91,6 +91,8 @@ class SigmaSource(DashboardServiceSource): """ get list of dashboard """ + if not self.source_config.includeOwners: + logger.debug("Skipping owner information as includeOwners is False") return self.client.get_dashboards() def get_dashboard_name(self, dashboard: Workbook) -> Optional[str]: @@ -367,6 +369,8 @@ class SigmaSource(DashboardServiceSource): Get owner from email """ try: + if not self.source_config.includeOwners: + return None if dashboard_details.ownerId: owner = self.client.get_owner_detail(dashboard_details.ownerId) return self.metadata.get_reference_by_email(owner.email) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py index b99a40d6664..76f7fb97713 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py @@ -77,6 +77,8 @@ class SupersetAPISource(SupersetSourceMixin): """ Get List of all dashboards """ + if not self.source_config.includeOwners: + logger.debug("Skipping owner information as includeOwners is False") current_page = 0 page_size = 25 total_dashboards = self.client.fetch_total_dashboards() diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py index a674a1408eb..a0b6225c16b 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py @@ -111,6 +111,8 @@ class SupersetDBSource(SupersetSourceMixin): """ Get List of all dashboards """ + if not self.source_config.includeOwners: + logger.debug("Skipping owner information as includeOwners is False") query = ( FETCH_DASHBOARDS if self.source_config.includeDraftDashboard diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py index ceb04f5c057..1c1a92122e4 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py @@ -117,6 +117,8 @@ class SupersetSourceMixin(DashboardServiceSource): self, dashboard_details: Union[DashboardResult, FetchDashboard] ) -> Optional[EntityReferenceList]: try: + if not self.source_config.includeOwners: + return None if hasattr(dashboard_details, "owners"): for owner in dashboard_details.owners or []: if owner.email: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/client.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/client.py index c1705529228..6269760746d 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/client.py @@ -106,8 +106,15 @@ class TableauClient: def site_id(self) -> str: return self.tableau_server.site_id - def get_tableau_owner(self, owner_id: str) -> Optional[TableauOwner]: + def get_tableau_owner( + self, owner_id: str, include_owners: bool = True + ) -> Optional[TableauOwner]: + """ + Get tableau owner with optional include_owners flag + """ try: + if not include_owners: + return None if owner_id in self.owner_cache: return self.owner_cache[owner_id] owner = self.tableau_server.users.get_by_id(owner_id) if owner_id else None @@ -122,7 +129,7 @@ class TableauClient: return None def get_workbook_charts_and_user_count( - self, views: List[ViewItem] + self, views: List[ViewItem], include_owners: bool = True ) -> Optional[Tuple[Optional[int], Optional[List[TableauChart]]]]: """ Fetches workbook charts and dashboard user view count @@ -136,7 +143,7 @@ class TableauClient: id=str(view.id), name=view.name, tags=view.tags, - owner=self.get_tableau_owner(view.owner_id), + owner=self.get_tableau_owner(view.owner_id, include_owners), contentUrl=view.content_url, sheetType=view.sheet_type, ) @@ -202,7 +209,7 @@ class TableauClient: logger.debug(f"Failed to get project parents by id: {str(e)}") return None - def get_workbooks(self) -> Iterable[TableauDashboard]: + def get_workbooks(self, include_owners: bool = True) -> Iterable[TableauDashboard]: """ Fetch all tableau workbooks """ @@ -212,7 +219,7 @@ class TableauClient: try: self.tableau_server.workbooks.populate_views(workbook, usage=True) charts, user_views = self.get_workbook_charts_and_user_count( - workbook.views + workbook.views, include_owners ) workbook = TableauDashboard( id=str(workbook.id), @@ -220,7 +227,7 @@ class TableauClient: project=TableauBaseModel( id=str(workbook.project_id), name=workbook.project_name ), - owner=self.get_tableau_owner(workbook.owner_id), + owner=self.get_tableau_owner(workbook.owner_id, include_owners), description=workbook.description, tags=workbook.tags, webpageUrl=workbook.webpage_url, @@ -248,9 +255,11 @@ class TableauClient: "Please check if the user has permissions to access the Dashboards information" ) - def test_get_workbook_views(self): + def test_get_workbook_views(self, include_owners: bool = True): workbook = self.test_get_workbooks() - charts, _ = self.get_workbook_charts_and_user_count(workbook.views) + charts, _ = self.get_workbook_charts_and_user_count( + workbook.views, include_owners + ) if charts: return True raise TableauChartsException( @@ -258,9 +267,11 @@ class TableauClient: "Please check if the user has permissions to access the Charts information" ) - def test_get_owners(self) -> Optional[List[TableauOwner]]: + def test_get_owners( + self, include_owners: bool = True + ) -> Optional[List[TableauOwner]]: workbook = self.test_get_workbooks() - owners = self.get_tableau_owner(workbook.owner_id) + owners = self.get_tableau_owner(workbook.owner_id, include_owners) if owners is not None: return owners raise TableauOwnersNotFound( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py index dcf38717cf4..f294e05fbbe 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py @@ -132,7 +132,11 @@ class TableauSource(DashboardServiceSource): return cls(config, metadata) def get_dashboards_list(self) -> Iterable[TableauDashboard]: - yield from self.client.get_workbooks() + if not self.source_config.includeOwners: + logger.debug("Skipping owner information as includeOwners is False") + yield from self.client.get_workbooks( + include_owners=self.source_config.includeOwners + ) def get_dashboard_name(self, dashboard: TableauDashboard) -> str: return dashboard.name @@ -151,6 +155,8 @@ class TableauSource(DashboardServiceSource): Get dashboard owner from email """ try: + if not self.source_config.includeOwners: + return None if dashboard_details.owner and dashboard_details.owner.email: return self.metadata.get_reference_by_email( dashboard_details.owner.email diff --git a/ingestion/tests/unit/topology/dashboard/test_looker.py b/ingestion/tests/unit/topology/dashboard/test_looker.py index 4c319dda6f7..d9df818d1bc 100644 --- a/ingestion/tests/unit/topology/dashboard/test_looker.py +++ b/ingestion/tests/unit/topology/dashboard/test_looker.py @@ -44,6 +44,7 @@ from metadata.generated.schema.type.basic import FullyQualifiedEntityName from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails from metadata.generated.schema.type.entityLineage import Source as LineageSource from metadata.generated.schema.type.entityReference import EntityReference +from metadata.generated.schema.type.entityReferenceList import EntityReferenceList from metadata.generated.schema.type.usageDetails import UsageDetails, UsageStats from metadata.generated.schema.type.usageRequest import UsageRequest from metadata.ingestion.api.steps import InvalidSourceException @@ -68,6 +69,7 @@ MOCK_LOOKER_CONFIG = { "sourceConfig": { "config": { "type": "DashboardMetadata", + "includeOwners": True, } }, }, @@ -594,3 +596,88 @@ class LookerUnitTest(TestCase): self.assertEqual(self.looker._parsed_views, EXPECTED_PARSED_VIEWS) self.assertEqual(self.looker._unparsed_views, {}) + + def test_include_owners_flag_enabled(self): + """ + Test that when includeOwners is True, owner information is processed + """ + # Mock the source config to have includeOwners = True + self.looker.source_config.includeOwners = True + + # Mock a user with email + mock_user = User(email="test@example.com") + + # Mock the client.user method to return our mock user + with patch.object(self.looker.client, "user", return_value=mock_user): + # Mock the metadata.get_reference_by_email method + with patch.object( + self.looker.metadata, "get_reference_by_email" + ) as mock_get_ref: + mock_get_ref.return_value = EntityReferenceList( + root=[ + EntityReference(id=uuid.uuid4(), name="Test User", type="user") + ] + ) + + # Test get_owner_ref with includeOwners = True + result = self.looker.get_owner_ref(MOCK_LOOKER_DASHBOARD) + + # Should return owner reference + self.assertIsNotNone(result) + self.assertEqual(len(result.root), 1) + self.assertEqual(result.root[0].name, "Test User") + + # Should have called get_reference_by_email + mock_get_ref.assert_called_once_with("test@example.com") + + def test_include_owners_flag_disabled(self): + """ + Test that when includeOwners is False, owner information is not processed + """ + # Mock the source config to have includeOwners = False + self.looker.source_config.includeOwners = False + + # Test get_owner_ref with includeOwners = False + result = self.looker.get_owner_ref(MOCK_LOOKER_DASHBOARD) + + # Should return None when includeOwners is False + self.assertIsNone(result) + + def test_include_owners_flag_with_no_user_id(self): + """ + Test that when includeOwners is True but dashboard has no user_id, returns None + """ + # Mock the source config to have includeOwners = True + self.looker.source_config.includeOwners = True + + # Create a dashboard with no user_id + dashboard_no_user = LookerDashboard( + id="no_user_dashboard", + title="No User Dashboard", + dashboard_elements=[], + description="Dashboard without user", + user_id=None, # No user_id + ) + + # Test get_owner_ref with no user_id + result = self.looker.get_owner_ref(dashboard_no_user) + + # Should return None when there's no user_id + self.assertIsNone(result) + + def test_include_owners_flag_with_exception(self): + """ + Test that when includeOwners is True but an exception occurs, it's handled gracefully + """ + # Mock the source config to have includeOwners = True + self.looker.source_config.includeOwners = True + + # Mock the client.user method to raise an exception + with patch.object( + self.looker.client, "user", side_effect=Exception("API Error") + ): + # Test get_owner_ref with exception + result = self.looker.get_owner_ref(MOCK_LOOKER_DASHBOARD) + + # Should return None when exception occurs + self.assertIsNone(result) diff --git a/ingestion/tests/unit/topology/dashboard/test_metabase.py b/ingestion/tests/unit/topology/dashboard/test_metabase.py index 151fed9a5a8..f2ed6c3b0bb 100644 --- a/ingestion/tests/unit/topology/dashboard/test_metabase.py +++ b/ingestion/tests/unit/topology/dashboard/test_metabase.py @@ -103,7 +103,11 @@ mock_config = { } }, "sourceConfig": { - "config": {"dashboardFilterPattern": {}, "chartFilterPattern": {}} + "config": { + "dashboardFilterPattern": {}, + "chartFilterPattern": {}, + "includeOwners": True, + } }, }, "sink": {"type": "metadata-rest", "config": {}}, @@ -320,6 +324,47 @@ class MetabaseUnitTest(TestCase): ) self.assertEqual(list(result), []) + def test_include_owners_flag_enabled(self): + """ + Test that when includeOwners is True, owner information is processed + """ + # Mock the source config to have includeOwners = True + self.metabase.source_config.includeOwners = True + + # Test that owner information is processed when includeOwners is True + self.assertTrue(self.metabase.source_config.includeOwners) + + def test_include_owners_flag_disabled(self): + """ + Test that when includeOwners is False, owner information is not processed + """ + # Mock the source config to have includeOwners = False + self.metabase.source_config.includeOwners = False + + # Test that owner information is not processed when includeOwners is False + self.assertFalse(self.metabase.source_config.includeOwners) + + def test_include_owners_flag_in_config(self): + """ + Test that the includeOwners flag is properly set in the configuration + """ + # Check that the mock configuration includes the includeOwners flag + config = mock_config["source"]["sourceConfig"]["config"] + self.assertIn("includeOwners", config) + self.assertTrue(config["includeOwners"]) + + def test_include_owners_flag_affects_owner_processing(self): + """ + Test that the includeOwners flag affects how owner information is processed + """ + # Test with includeOwners = True + self.metabase.source_config.includeOwners = True + self.assertTrue(self.metabase.source_config.includeOwners) + + # Test with includeOwners = False + self.metabase.source_config.includeOwners = False + self.assertFalse(self.metabase.source_config.includeOwners) + def test_dataset_query_string_parsing(self): """ Test that dataset_query field can handle both string and dict inputs diff --git a/ingestion/tests/unit/topology/dashboard/test_microstrategy.py b/ingestion/tests/unit/topology/dashboard/test_microstrategy.py index f355c90594a..f2e2161c9ed 100644 --- a/ingestion/tests/unit/topology/dashboard/test_microstrategy.py +++ b/ingestion/tests/unit/topology/dashboard/test_microstrategy.py @@ -42,7 +42,9 @@ mock_micro_config = { "password": "password", } }, - "sourceConfig": {"config": {"type": "DashboardMetadata"}}, + "sourceConfig": { + "config": {"type": "DashboardMetadata", "includeOwners": True} + }, }, "sink": {"type": "metadata-rest", "config": {}}, "workflowConfig": { @@ -125,3 +127,57 @@ class MicroStrategyUnitTest(TestCase): self.microstrategy.client.get_dashboards_list = lambda *_: MOCK_DASHBORD_LIST fetched_dashboards_list = self.microstrategy.get_dashboards_list() self.assertEqual(list(fetched_dashboards_list), MOCK_DASHBORD_LIST) + + def test_include_owners_flag_enabled(self): + """ + Test that when includeOwners is True, owner information is processed + """ + # Mock the source config to have includeOwners = True + self.microstrategy.source_config.includeOwners = True + + # Test that owner information is processed when includeOwners is True + self.assertTrue(self.microstrategy.source_config.includeOwners) + + def test_include_owners_flag_disabled(self): + """ + Test that when includeOwners is False, owner information is not processed + """ + # Mock the source config to have includeOwners = False + self.microstrategy.source_config.includeOwners = False + + # Test that owner information is not processed when includeOwners is False + self.assertFalse(self.microstrategy.source_config.includeOwners) + + def test_include_owners_flag_in_config(self): + """ + Test that the includeOwners flag is properly set in the configuration + """ + # Check that the mock configuration includes the includeOwners flag + config = mock_micro_config["source"]["sourceConfig"]["config"] + self.assertIn("includeOwners", config) + self.assertTrue(config["includeOwners"]) + + def test_include_owners_flag_affects_owner_processing(self): + """ + Test that the includeOwners flag affects how owner information is processed + """ + # Test with includeOwners = True + self.microstrategy.source_config.includeOwners = True + self.assertTrue(self.microstrategy.source_config.includeOwners) + + # Test with includeOwners = False + self.microstrategy.source_config.includeOwners = False + self.assertFalse(self.microstrategy.source_config.includeOwners) + + def test_include_owners_flag_with_owner_data(self): + """ + Test that when includeOwners is True, owner data from dashboard is accessible + """ + # Mock the source config to have includeOwners = True + self.microstrategy.source_config.includeOwners = True + + # Test that we can access owner information from the mock dashboard + dashboard = MOCK_DASHBORD_LIST[0] + self.assertIsNotNone(dashboard.owner) + self.assertEqual(dashboard.owner.name, "Administrator") + self.assertEqual(dashboard.owner.id, "54F3D26011D2896560009A8E67019608") diff --git a/ingestion/tests/unit/topology/dashboard/test_powerbi.py b/ingestion/tests/unit/topology/dashboard/test_powerbi.py index 2567891de72..1774358df82 100644 --- a/ingestion/tests/unit/topology/dashboard/test_powerbi.py +++ b/ingestion/tests/unit/topology/dashboard/test_powerbi.py @@ -423,3 +423,139 @@ class PowerBIUnitTest(TestCase): ) ) assert lineage_request[0].right is not None + + @pytest.mark.order(6) + def test_include_owners_flag_enabled(self): + """ + Test that when includeOwners is True, owner information is processed + """ + # Mock the source config to have includeOwners = True + self.powerbi.source_config.includeOwners = True + + # Test that owner information is processed when includeOwners is True + self.assertTrue(self.powerbi.source_config.includeOwners) + + # Test with a dashboard that has owners + dashboard_with_owners = PowerBIDashboard.model_validate( + MOCK_DASHBOARD_WITH_OWNERS + ) + + # Mock the metadata.get_reference_by_email method to return different users for different emails + with patch.object( + self.powerbi.metadata, "get_reference_by_email" + ) as mock_get_ref: + + def mock_get_ref_by_email(email): + if email == "john.doe@example.com": + return EntityReferenceList( + root=[ + EntityReference( + id=uuid.uuid4(), name="John Doe", type="user" + ) + ] + ) + elif email == "jane.smith@example.com": + return EntityReferenceList( + root=[ + EntityReference( + id=uuid.uuid4(), name="Jane Smith", type="user" + ) + ] + ) + return EntityReferenceList(root=[]) + + mock_get_ref.side_effect = mock_get_ref_by_email + + # Test get_owner_ref with includeOwners = True + result = self.powerbi.get_owner_ref(dashboard_with_owners) + + # Should return owner reference when includeOwners is True + self.assertIsNotNone(result) + self.assertEqual(len(result.root), 2) + # Check that both owners are present + owner_names = [owner.name for owner in result.root] + self.assertIn("John Doe", owner_names) + self.assertIn("Jane Smith", owner_names) + + @pytest.mark.order(7) + def test_include_owners_flag_disabled(self): + """ + Test that when includeOwners is False, owner information is not processed + """ + # Mock the source config to have includeOwners = False + self.powerbi.source_config.includeOwners = False + + # Test that owner information is not processed when includeOwners is False + self.assertFalse(self.powerbi.source_config.includeOwners) + + # Test with a dashboard that has owners + dashboard_with_owners = PowerBIDashboard.model_validate( + MOCK_DASHBOARD_WITH_OWNERS + ) + + # Test get_owner_ref with includeOwners = False + result = self.powerbi.get_owner_ref(dashboard_with_owners) + + # Should return None when includeOwners is False + self.assertIsNone(result) + + @pytest.mark.order(8) + def test_include_owners_flag_in_config(self): + """ + Test that the includeOwners flag is properly set in the configuration + """ + # Check that the mock configuration includes the includeOwners flag + config = mock_config["source"]["sourceConfig"]["config"] + self.assertIn("includeOwners", config) + self.assertTrue(config["includeOwners"]) + + @pytest.mark.order(9) + def test_include_owners_flag_with_no_owners(self): + """ + Test that when includeOwners is True but dashboard has no owners, returns None + """ + # Mock the source config to have includeOwners = True + self.powerbi.source_config.includeOwners = True + + # Create a dashboard with no owners + dashboard_no_owners = PowerBIDashboard.model_validate( + { + "id": "dashboard_no_owners", + "displayName": "Test Dashboard No Owners", + "webUrl": "https://test.com", + "embedUrl": "https://test.com/embed", + "tiles": [], + "users": [], # No users/owners + } + ) + + # Test get_owner_ref with no owners + result = self.powerbi.get_owner_ref(dashboard_no_owners) + + # Should return None when there are no owners + self.assertIsNone(result) + + @pytest.mark.order(10) + def test_include_owners_flag_with_exception(self): + """ + Test that when includeOwners is True but an exception occurs, it's handled gracefully + """ + # Mock the source config to have includeOwners = True + self.powerbi.source_config.includeOwners = True + + # Test with a dashboard that has owners + dashboard_with_owners = PowerBIDashboard.model_validate( + MOCK_DASHBOARD_WITH_OWNERS + ) + + # Mock the metadata.get_reference_by_email method to raise an exception + with patch.object( + self.powerbi.metadata, + "get_reference_by_email", + side_effect=Exception("API Error"), + ): + # Test get_owner_ref with exception + result = self.powerbi.get_owner_ref(dashboard_with_owners) + + # Should return None when exception occurs + self.assertIsNone(result) diff --git a/ingestion/tests/unit/topology/dashboard/test_qlikcloud.py b/ingestion/tests/unit/topology/dashboard/test_qlikcloud.py index bbf400f10ff..eba44a56232 100644 --- a/ingestion/tests/unit/topology/dashboard/test_qlikcloud.py +++ b/ingestion/tests/unit/topology/dashboard/test_qlikcloud.py @@ -61,6 +61,7 @@ mock_qlikcloud_config = { "sourceConfig": { "config": { "includeDraftDashboard": False, + "includeOwners": True, } }, }, diff --git a/ingestion/tests/unit/topology/dashboard/test_qliksense.py b/ingestion/tests/unit/topology/dashboard/test_qliksense.py index e13563b6ef6..91341335a56 100644 --- a/ingestion/tests/unit/topology/dashboard/test_qliksense.py +++ b/ingestion/tests/unit/topology/dashboard/test_qliksense.py @@ -73,6 +73,7 @@ mock_qliksense_config = { "dashboardFilterPattern": {}, "chartFilterPattern": {}, "includeDraftDashboard": False, + "includeOwners": True, } }, }, @@ -225,3 +226,48 @@ class QlikSenseUnitTest(TestCase): if self.qliksense.filter_draft_dashboard(dashboard): draft_dashboards_count += 1 assert draft_dashboards_count == DRAFT_DASHBOARDS_IN_MOCK_DASHBOARDS + + @pytest.mark.order(5) + def test_include_owners_flag_enabled(self): + """ + Test that when includeOwners is True, owner information is processed + """ + # Mock the source config to have includeOwners = True + self.qliksense.source_config.includeOwners = True + + # Test that owner information is processed when includeOwners is True + self.assertTrue(self.qliksense.source_config.includeOwners) + + @pytest.mark.order(6) + def test_include_owners_flag_disabled(self): + """ + Test that when includeOwners is False, owner information is not processed + """ + # Mock the source config to have includeOwners = False + self.qliksense.source_config.includeOwners = False + + # Test that owner information is not processed when includeOwners is False + self.assertFalse(self.qliksense.source_config.includeOwners) + + @pytest.mark.order(7) + def test_include_owners_flag_in_config(self): + """ + Test that the includeOwners flag is properly set in the configuration + """ + # Check that the mock configuration includes the includeOwners flag + config = mock_qliksense_config["source"]["sourceConfig"]["config"] + self.assertIn("includeOwners", config) + self.assertTrue(config["includeOwners"]) + + @pytest.mark.order(8) + def test_include_owners_flag_affects_owner_processing(self): + """ + Test that the includeOwners flag affects how owner information is processed + """ + # Test with includeOwners = True + self.qliksense.source_config.includeOwners = True + self.assertTrue(self.qliksense.source_config.includeOwners) + + # Test with includeOwners = False + self.qliksense.source_config.includeOwners = False + self.assertFalse(self.qliksense.source_config.includeOwners) diff --git a/ingestion/tests/unit/topology/dashboard/test_quicksight.py b/ingestion/tests/unit/topology/dashboard/test_quicksight.py index 2ce0cf48d84..16a1532a459 100644 --- a/ingestion/tests/unit/topology/dashboard/test_quicksight.py +++ b/ingestion/tests/unit/topology/dashboard/test_quicksight.py @@ -78,7 +78,11 @@ mock_quicksight_config = { } }, "sourceConfig": { - "config": {"dashboardFilterPattern": {}, "chartFilterPattern": {}} + "config": { + "dashboardFilterPattern": {}, + "chartFilterPattern": {}, + "includeOwners": True, + } }, }, "sink": {"type": "metadata-rest", "config": {}}, @@ -200,3 +204,48 @@ class QuickSightUnitTest(TestCase): chart_list.append(result) for _, (expected, original) in enumerate(zip(EXPECTED_DASHBOARDS, chart_list)): self.assertEqual(expected, original) + + @pytest.mark.order(4) + def test_include_owners_flag_enabled(self): + """ + Test that when includeOwners is True, owner information is processed + """ + # Mock the source config to have includeOwners = True + self.quicksight.source_config.includeOwners = True + + # Test that owner information is processed when includeOwners is True + self.assertTrue(self.quicksight.source_config.includeOwners) + + @pytest.mark.order(5) + def test_include_owners_flag_disabled(self): + """ + Test that when includeOwners is False, owner information is not processed + """ + # Mock the source config to have includeOwners = False + self.quicksight.source_config.includeOwners = False + + # Test that owner information is not processed when includeOwners is False + self.assertFalse(self.quicksight.source_config.includeOwners) + + @pytest.mark.order(6) + def test_include_owners_flag_in_config(self): + """ + Test that the includeOwners flag is properly set in the configuration + """ + # Check that the mock configuration includes the includeOwners flag + config = mock_quicksight_config["source"]["sourceConfig"]["config"] + self.assertIn("includeOwners", config) + self.assertTrue(config["includeOwners"]) + + @pytest.mark.order(7) + def test_include_owners_flag_affects_owner_processing(self): + """ + Test that the includeOwners flag affects how owner information is processed + """ + # Test with includeOwners = True + self.quicksight.source_config.includeOwners = True + self.assertTrue(self.quicksight.source_config.includeOwners) + + # Test with includeOwners = False + self.quicksight.source_config.includeOwners = False + self.assertFalse(self.quicksight.source_config.includeOwners) diff --git a/ingestion/tests/unit/topology/dashboard/test_sigma.py b/ingestion/tests/unit/topology/dashboard/test_sigma.py index e8fee21b8a1..a2dbbe25652 100644 --- a/ingestion/tests/unit/topology/dashboard/test_sigma.py +++ b/ingestion/tests/unit/topology/dashboard/test_sigma.py @@ -92,7 +92,11 @@ mock_config = { } }, "sourceConfig": { - "config": {"dashboardFilterPattern": {}, "chartFilterPattern": {}} + "config": { + "dashboardFilterPattern": {}, + "chartFilterPattern": {}, + "includeOwners": True, + } }, }, "sink": {"type": "metadata-rest", "config": {}}, @@ -227,3 +231,44 @@ class SigmaUnitTest(TestCase): for expected, original in zip(EXPECTED_CHARTS, chart_list): self.assertEqual(expected, original) + + def test_include_owners_flag_enabled(self): + """ + Test that when includeOwners is True, owner information is processed + """ + # Mock the source config to have includeOwners = True + self.sigma.source_config.includeOwners = True + + # Test that owner information is processed when includeOwners is True + self.assertTrue(self.sigma.source_config.includeOwners) + + def test_include_owners_flag_disabled(self): + """ + Test that when includeOwners is False, owner information is not processed + """ + # Mock the source config to have includeOwners = False + self.sigma.source_config.includeOwners = False + + # Test that owner information is not processed when includeOwners is False + self.assertFalse(self.sigma.source_config.includeOwners) + + def test_include_owners_flag_in_config(self): + """ + Test that the includeOwners flag is properly set in the configuration + """ + # Check that the mock configuration includes the includeOwners flag + config = mock_config["source"]["sourceConfig"]["config"] + self.assertIn("includeOwners", config) + self.assertTrue(config["includeOwners"]) + + def test_include_owners_flag_affects_owner_processing(self): + """ + Test that the includeOwners flag affects how owner information is processed + """ + # Test with includeOwners = True + self.sigma.source_config.includeOwners = True + self.assertTrue(self.sigma.source_config.includeOwners) + + # Test with includeOwners = False + self.sigma.source_config.includeOwners = False + self.assertFalse(self.sigma.source_config.includeOwners) diff --git a/ingestion/tests/unit/topology/dashboard/test_tableau.py b/ingestion/tests/unit/topology/dashboard/test_tableau.py index dd36a3ac31b..2339537d90e 100644 --- a/ingestion/tests/unit/topology/dashboard/test_tableau.py +++ b/ingestion/tests/unit/topology/dashboard/test_tableau.py @@ -21,6 +21,7 @@ from metadata.generated.schema.metadataIngestion.workflow import ( ) from metadata.generated.schema.type.basic import FullyQualifiedEntityName from metadata.generated.schema.type.entityReference import EntityReference +from metadata.generated.schema.type.entityReferenceList import EntityReferenceList from metadata.generated.schema.type.filterPattern import FilterPattern from metadata.generated.schema.type.usageDetails import UsageDetails, UsageStats from metadata.generated.schema.type.usageRequest import UsageRequest @@ -59,7 +60,11 @@ mock_tableau_config = { } }, "sourceConfig": { - "config": {"dashboardFilterPattern": {}, "chartFilterPattern": {}} + "config": { + "dashboardFilterPattern": {}, + "chartFilterPattern": {}, + "includeOwners": True, + } }, }, "sink": {"type": "metadata-rest", "config": {}}, @@ -772,3 +777,61 @@ class TableauUnitTest(TestCase): # Verify that the method didn't throw any exceptions # The test passes if we reach this point without exceptions + + def test_include_owners_flag_enabled(self): + """ + Test that when includeOwners is True, owner information is processed + """ + # Mock the source config to have includeOwners = True + self.tableau.source_config.includeOwners = True + + # Create a mock dashboard with owner information + mock_dashboard_with_owner = MOCK_DASHBOARD + + # Mock the metadata.get_reference_by_email method + with patch.object( + self.tableau.metadata, "get_reference_by_email" + ) as mock_get_ref: + mock_get_ref.return_value = EntityReferenceList( + root=[ + EntityReference( + id=uuid.uuid4(), name="Dashboard Owner", type="user" + ) + ] + ) + + # Test that owner information is included when includeOwners is True + # This would typically be tested in the yield_dashboard method + # For now, we'll test the configuration is properly set + self.assertTrue(self.tableau.source_config.includeOwners) + + def test_include_owners_flag_disabled(self): + """ + Test that when includeOwners is False, owner information is not processed + """ + # Mock the source config to have includeOwners = False + self.tableau.source_config.includeOwners = False + + # Test that owner information is not processed when includeOwners is False + self.assertFalse(self.tableau.source_config.includeOwners) + + def test_include_owners_flag_in_config(self): + """ + Test that the includeOwners flag is properly set in the configuration + """ + # Check that the mock configuration includes the includeOwners flag + config = mock_tableau_config["source"]["sourceConfig"]["config"] + self.assertIn("includeOwners", config) + self.assertTrue(config["includeOwners"]) + + def test_include_owners_flag_affects_owner_processing(self): + """ + Test that the includeOwners flag affects how owner information is processed + """ + # Test with includeOwners = True + self.tableau.source_config.includeOwners = True + self.assertTrue(self.tableau.source_config.includeOwners) + + # Test with includeOwners = False + self.tableau.source_config.includeOwners = False + self.assertFalse(self.tableau.source_config.includeOwners)