Fix e2e for tableau, metabase and powerbi (#13050)

* Fix e2e * Fix e2e
2026-01-07 21:16:45 +00:00 · 2023-08-31 15:45:41 +02:00 · 2023-08-31 15:45:41 +02:00 · 1188831cd5
commit 1188831cd5
parent 8c223d0ccb
12 changed files with 71 additions and 54 deletions
--- a/ingestion/tests/cli_e2e/common/test_cli_dashboard.py
+++ b/ingestion/tests/cli_e2e/common/test_cli_dashboard.py
@ -49,16 +49,26 @@ class CliCommonDashboard:
            )

        def assert_not_including(self, source_status: Status, sink_status: Status):
+            """
+            Here we can have a diff of 1 element due to the service
+            being ingested in the first round.
+
+            This will not happen on subsequent tests or executions
+            """
            self.assertTrue(len(source_status.failures) == 0)
            self.assertTrue(len(source_status.warnings) == 0)
            self.assertTrue(len(source_status.filtered) == 0)
-            self.assertEqual(
-                self.expected_not_included_entities(), len(source_status.records)
+            # We can have a diff of 1 element if we are counting the service, which is only marked as ingested in the
+            # first go
+            self.assertTrue(
+                self.expected_dashboards_and_charts() + self.expected_lineage()
+                <= len(source_status.records)
            )
            self.assertTrue(len(sink_status.failures) == 0)
            self.assertTrue(len(sink_status.warnings) == 0)
-            self.assertEqual(
-                self.expected_not_included_sink_entities(), len(sink_status.records)
+            self.assertTrue(
+                self.expected_dashboards_and_charts() + self.expected_lineage()
+                <= len(sink_status.records)
            )

        def assert_for_vanilla_ingestion(
@ -67,14 +77,23 @@ class CliCommonDashboard:
            self.assertTrue(len(source_status.failures) == 0)
            self.assertTrue(len(source_status.warnings) == 0)
            self.assertTrue(len(source_status.filtered) == 0)
-            self.assertEqual(len(source_status.records), self.expected_entities())
+            self.assertEqual(
+                len(source_status.records),
+                self.expected_dashboards_and_charts()
+                + self.expected_tags()
+                + self.expected_lineage()
+                + self.expected_datamodels()
+                + self.expected_datamodel_lineage(),
+            )
            self.assertTrue(len(sink_status.failures) == 0)
            self.assertTrue(len(sink_status.warnings) == 0)
            self.assertEqual(
                len(sink_status.records),
-                self.expected_entities()
+                self.expected_dashboards_and_charts()
                + self.expected_tags()
-                + self.expected_lineage(),
+                + self.expected_lineage()
+                + self.expected_datamodels()
+                + self.expected_datamodel_lineage(),
            )

        def assert_filtered_mix(self, source_status: Status, sink_status: Status):
@ -89,7 +108,7 @@ class CliCommonDashboard:

        @staticmethod
        @abstractmethod
-        def expected_entities() -> int:
+        def expected_dashboards_and_charts() -> int:
            raise NotImplementedError()

        @staticmethod
@ -104,12 +123,12 @@ class CliCommonDashboard:

        @staticmethod
        @abstractmethod
-        def expected_not_included_entities() -> int:
+        def expected_datamodels() -> int:
            raise NotImplementedError()

        @staticmethod
        @abstractmethod
-        def expected_not_included_sink_entities() -> int:
+        def expected_datamodel_lineage() -> int:
            raise NotImplementedError()

        @staticmethod
--- a/ingestion/tests/cli_e2e/dashboard/metabase/metabase.yaml
+++ b/ingestion/tests/cli_e2e/dashboard/metabase/metabase.yaml
@ -13,7 +13,7 @@ source:
      dashboardFilterPattern: {}
      chartFilterPattern: {}
      dbServiceNames:
-      - local_redshift
+      - local_redshift_metabase
 sink:
  type: metadata-rest
  config: {}
--- a/ingestion/tests/cli_e2e/dashboard/metabase/redshift.yaml
+++ b/ingestion/tests/cli_e2e/dashboard/metabase/redshift.yaml
@ -1,6 +1,6 @@
 source:
  type: redshift
-  serviceName: local_redshift
+  serviceName: local_redshift_metabase
  serviceConnection:
    config:
      hostPort: $E2E_REDSHIFT_HOST_PORT
--- a/ingestion/tests/cli_e2e/dashboard/powerbi/powerbi.yaml
+++ b/ingestion/tests/cli_e2e/dashboard/powerbi/powerbi.yaml
@ -17,7 +17,7 @@ source:
      dashboardFilterPattern: {}
      chartFilterPattern: {}
      dbServiceNames:
-      - local_redshift
+      - local_redshift_powerbi
 sink:
  type: metadata-rest
  config: {}
--- a/ingestion/tests/cli_e2e/dashboard/powerbi/redshift.yaml
+++ b/ingestion/tests/cli_e2e/dashboard/powerbi/redshift.yaml
@ -1,6 +1,6 @@
 source:
  type: redshift
-  serviceName: local_redshift
+  serviceName: local_redshift_powerbi
  serviceConnection:
    config:
      hostPort: $E2E_REDSHIFT_HOST_PORT
--- a/ingestion/tests/cli_e2e/dashboard/tableau/redshift.yaml
+++ b/ingestion/tests/cli_e2e/dashboard/tableau/redshift.yaml
@ -1,6 +1,6 @@
 source:
  type: redshift
-  serviceName: local_redshift
+  serviceName: local_redshift_tableau
  serviceConnection:
    config:
      hostPort: $E2E_REDSHIFT_HOST_PORT
--- a/ingestion/tests/cli_e2e/dashboard/tableau/tableau.yaml
+++ b/ingestion/tests/cli_e2e/dashboard/tableau/tableau.yaml
@ -16,7 +16,7 @@ source:
    config:
      type: DashboardMetadata
      dbServiceNames:
-      - local_redshift
+      - local_redshift_tableau
 sink:
  type: metadata-rest
  config: {}
--- a/ingestion/tests/cli_e2e/test_cli_metabase.py
+++ b/ingestion/tests/cli_e2e/test_cli_metabase.py
@ -55,7 +55,13 @@ class MetabaseCliTest(CliCommonDashboard.TestSuite):
    def get_excludes_datamodels(self) -> List[str]:
        return []

-    def expected_entities(self) -> int:
+    def expected_datamodels(self) -> int:
+        return 0
+
+    def expected_datamodel_lineage(self) -> int:
+        return 0
+
+    def expected_dashboards_and_charts(self) -> int:
        return 6

    def expected_lineage(self) -> int:
@ -64,14 +70,8 @@ class MetabaseCliTest(CliCommonDashboard.TestSuite):
    def expected_tags(self) -> int:
        return 0

-    def expected_not_included_entities(self) -> int:
-        return 6
-
-    def expected_not_included_sink_entities(self) -> int:
-        return 12
-
    def expected_filtered_mix(self) -> int:
        return 2

    def expected_filtered_sink_mix(self) -> int:
-        return 5
+        return 6
--- a/ingestion/tests/cli_e2e/test_cli_powerbi.py
+++ b/ingestion/tests/cli_e2e/test_cli_powerbi.py
@ -47,31 +47,29 @@ class PowerBICliTest(CliCommonDashboard.TestSuite):
    def get_excludes_charts(self) -> List[str]:
        return ["Total Rejected Defect Quantity"]

-    # PowerBI do not ingest datamodels
    def get_includes_datamodels(self) -> List[str]:
        return []

-    # PowerBI do not ingest datamodels
    def get_excludes_datamodels(self) -> List[str]:
        return []

-    def expected_entities(self) -> int:
-        return 91
+    def expected_datamodels(self) -> int:
+        return 9
+
+    def expected_dashboards_and_charts(self) -> int:
+        return 83

    def expected_lineage(self) -> int:
-        return 58
+        return 44
+
+    def expected_datamodel_lineage(self) -> int:
+        return 13

    def expected_tags(self) -> int:
        return 0

-    def expected_not_included_entities(self) -> int:
-        return 82
-
-    def expected_not_included_sink_entities(self) -> int:
-        return 127
-
    def expected_filtered_mix(self) -> int:
        return 19

    def expected_filtered_sink_mix(self) -> int:
-        return 36
+        return 46
--- a/ingestion/tests/cli_e2e/test_cli_quicksight.py
+++ b/ingestion/tests/cli_e2e/test_cli_quicksight.py
@ -34,7 +34,7 @@ class QuicksightCliTest(CliCommonDashboard.TestSuite):
    def get_excludes_charts(self) -> List[str]:
        return []

-    def expected_entities(self) -> int:
+    def expected_dashboards_and_charts(self) -> int:
        return 6

    def expected_lineage(self) -> int:
--- a/ingestion/tests/cli_e2e/test_cli_redash.py
+++ b/ingestion/tests/cli_e2e/test_cli_redash.py
@ -42,7 +42,7 @@ class RedashCliTest(CliCommonDashboard.TestSuite):
    def get_excludes_datamodels(self) -> List[str]:
        return []

-    def expected_entities(self) -> int:
+    def expected_dashboards_and_charts(self) -> int:
        return 12

    def expected_lineage(self) -> int:
@ -51,11 +51,11 @@ class RedashCliTest(CliCommonDashboard.TestSuite):
    def expected_tags(self) -> int:
        return 4

-    def expected_not_included_entities(self) -> int:
-        return 12
+    def expected_datamodels(self) -> int:
+        return 0

-    def expected_not_included_sink_entities(self) -> int:
-        return 13
+    def expected_datamodel_lineage(self) -> int:
+        return 0

    def expected_filtered_mix(self) -> int:
        return 6
--- a/ingestion/tests/cli_e2e/test_cli_tableau.py
+++ b/ingestion/tests/cli_e2e/test_cli_tableau.py
@ -53,20 +53,20 @@ class TableauCliTest(CliCommonDashboard.TestSuite):
    def get_excludes_datamodels(self) -> List[str]:
        return ["Random.*"]

-    def expected_entities(self) -> int:
-        return 30
-
-    def expected_lineage(self) -> int:
-        return 11
-
-    def expected_tags(self) -> int:
-        return 2
-
-    def expected_not_included_entities(self) -> int:
+    def expected_dashboards_and_charts(self) -> int:
        return 20

-    def expected_not_included_sink_entities(self) -> int:
-        return 21
+    def expected_lineage(self) -> int:
+        return 0
+
+    def expected_tags(self) -> int:
+        return 1
+
+    def expected_datamodel_lineage(self) -> int:
+        return 11
+
+    def expected_datamodels(self) -> int:
+        return 10

    def expected_filtered_mix(self) -> int:
        return 12