Add policy tags from Bigquery (#10189)

.gitignore

@@ -71,6 +71,7 @@ openmetadata-ui/src/main/resources/ui/tsconfig.tsbuildinfo
 
 #tests
 .coverage*
+!ingestion/.coveragerc
 /ingestion/coverage.xml
 /ingestion/ci-coverage.xml
 /ingestion/junit/*

bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql

@@ -1,3 +1,8 @@
 -- Unique constraint for user email address
 ALTER TABLE user_entity
-ADD UNIQUE (email);
+ADD UNIQUE (email);
+
+
+-- Remove classificationName in BigQuery
+UPDATE dbservice_entity
+SET json = JSON_REMOVE(json, '$.connection.config.classificationName') where serviceType in ('BigQuery');

bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql

@@ -1,3 +1,7 @@
 -- Unique constraint for user email address
 ALTER TABLE user_entity
 ADD UNIQUE (email);
+
+
+-- Remove classificationName in BigQuery
+UPDATE dbservice_entity SET json = json #- '{connection,config,classificationName}' where serviceType in ('BigQuery');

docker/local-metadata/docker-compose-postgres.yml

@@ -64,6 +64,7 @@ services:
       dockerfile: docker/local-metadata/Dockerfile
     container_name: openmetadata_server
     environment:
+      OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false}
       # OpenMetadata Server Authentication Configuration
       AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
       AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
@@ -111,9 +112,11 @@ services:
     expose:
       - 8585
       - 8586
+      - 5005
     ports:
       - "8585:8585"
       - "8586:8586"
+      - "5005:5005"
     depends_on:
       elasticsearch:
         condition: service_started
@@ -131,13 +134,6 @@ services:
       args:
         INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all}
     container_name: openmetadata_ingestion
-    depends_on:
-      elasticsearch:
-        condition: service_started
-      postgresql:
-        condition: service_healthy
-      openmetadata-server:
-        condition: service_started
     environment:
       AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
       AIRFLOW__CORE__EXECUTOR: LocalExecutor
@@ -151,6 +147,13 @@ services:
     entrypoint: /bin/bash
     command:
       - "/opt/airflow/ingestion_dependency.sh"
+    depends_on:
+      elasticsearch:
+        condition: service_started
+      postgresql:
+        condition: service_healthy
+      openmetadata-server:
+        condition: service_started
     expose:
       - 8080
     ports:
@@ -163,6 +166,7 @@ services:
       - ingestion-volume-tmp:/tmp
       - /var/run/docker.sock:/var/run/docker.sock:z  # Need 600 permissions to run DockerOperator
 
+
 networks:
   local_app_net:
     name: ometa_network

ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py

@@ -84,9 +84,22 @@ def get_columns(bq_schema):
         }
         try:
             if field.policy_tags:
+                policy_tag_name = field.policy_tags.names[0]
+                taxonomy_name = (
+                    policy_tag_name.split("/policyTags/")[0] if policy_tag_name else ""
+                )
+                if not taxonomy_name:
+                    raise NotImplementedError(
+                        f"Taxonomy Name not present for {field.name}"
+                    )
+                col_obj["taxonomy"] = (
+                    PolicyTagManagerClient()
+                    .get_taxonomy(name=taxonomy_name)
+                    .display_name
+                )
                 col_obj["policy_tags"] = (
                     PolicyTagManagerClient()
-                    .get_policy_tag(name=field.policy_tags.names[0])
+                    .get_policy_tag(name=policy_tag_name)
                     .display_name
                 )
         except Exception as exc:
@@ -162,11 +175,11 @@ class BigquerySource(CommonDbSourceService):
                     for tag in policy_tags:
                         yield OMetaTagAndClassification(
                             classification_request=CreateClassificationRequest(
-                                name=self.service_connection.classificationName,
+                                name=taxonomy.display_name,
                                 description="",
                             ),
                             tag_request=CreateTagRequest(
-                                classification=self.service_connection.classificationName,
+                                classification=taxonomy.display_name,
                                 name=tag.display_name,
                                 description="Bigquery Policy Tag",
                             ),
@@ -195,7 +208,7 @@ class BigquerySource(CommonDbSourceService):
                     tagFQN=fqn.build(
                         self.metadata,
                         entity_type=Tag,
-                        classification_name=self.service_connection.classificationName,
+                        classification_name=column["taxonomy"],
                         tag_name=column["policy_tags"],
                     ),
                     labelType="Automated",

ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py

@@ -50,7 +50,6 @@ from metadata.generated.schema.entity.services.pipelineService import PipelineSe
 from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
     OpenMetadataJWTClientConfig,
 )
-from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 
 OM_HOST_PORT = "http://localhost:8585/api"

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json

@@ -47,12 +47,6 @@
       "description": "GCS Credentials",
       "$ref": "../../../../security/credentials/gcsCredentials.json"
     },
-    "classificationName": {
-      "title": "Classification Name",
-      "description": "Custom OpenMetadata Classification name for BigQuery policy tags.",
-      "type": "string",
-      "default": "BigqueryPolicyTags"
-    },
     "partitionQueryDuration": {
       "title": "Partition Query Duration",
       "description": "Duration for partitioning BigQuery tables.",