diff --git a/.gitignore b/.gitignore index 972ae67c500..5c0fa1419e5 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ openmetadata-ui/src/main/resources/ui/tsconfig.tsbuildinfo #tests .coverage* +!ingestion/.coveragerc /ingestion/coverage.xml /ingestion/ci-coverage.xml /ingestion/junit/* diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql index b17fa9f80f8..296bb006b11 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql @@ -1,3 +1,8 @@ -- Unique constraint for user email address ALTER TABLE user_entity -ADD UNIQUE (email); \ No newline at end of file +ADD UNIQUE (email); + + +-- Remove classificationName in BigQuery +UPDATE dbservice_entity +SET json = JSON_REMOVE(json, '$.connection.config.classificationName') where serviceType in ('BigQuery'); \ No newline at end of file diff --git a/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql index e571a639c55..909c7ec4f37 100644 --- a/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql @@ -1,3 +1,7 @@ -- Unique constraint for user email address ALTER TABLE user_entity ADD UNIQUE (email); + + +-- Remove classificationName in BigQuery +UPDATE dbservice_entity SET json = json #- '{connection,config,classificationName}' where serviceType in ('BigQuery'); \ No newline at end of file diff --git a/docker/local-metadata/docker-compose-postgres.yml b/docker/local-metadata/docker-compose-postgres.yml index d1b6de61875..e40c9179eed 100644 --- a/docker/local-metadata/docker-compose-postgres.yml +++ b/docker/local-metadata/docker-compose-postgres.yml @@ -64,6 +64,7 @@ services: dockerfile: docker/local-metadata/Dockerfile container_name: openmetadata_server environment: + OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false} # OpenMetadata Server Authentication Configuration AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer} AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter} @@ -111,9 +112,11 @@ services: expose: - 8585 - 8586 + - 5005 ports: - "8585:8585" - "8586:8586" + - "5005:5005" depends_on: elasticsearch: condition: service_started @@ -131,13 +134,6 @@ services: args: INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all} container_name: openmetadata_ingestion - depends_on: - elasticsearch: - condition: service_started - postgresql: - condition: service_healthy - openmetadata-server: - condition: service_started environment: AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__CORE__EXECUTOR: LocalExecutor @@ -151,6 +147,13 @@ services: entrypoint: /bin/bash command: - "/opt/airflow/ingestion_dependency.sh" + depends_on: + elasticsearch: + condition: service_started + postgresql: + condition: service_healthy + openmetadata-server: + condition: service_started expose: - 8080 ports: @@ -163,6 +166,7 @@ services: - ingestion-volume-tmp:/tmp - /var/run/docker.sock:/var/run/docker.sock:z # Need 600 permissions to run DockerOperator + networks: local_app_net: name: ometa_network diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py index 69a4413f0ce..153b3a41fa8 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py @@ -84,9 +84,22 @@ def get_columns(bq_schema): } try: if field.policy_tags: + policy_tag_name = field.policy_tags.names[0] + taxonomy_name = ( + policy_tag_name.split("/policyTags/")[0] if policy_tag_name else "" + ) + if not taxonomy_name: + raise NotImplementedError( + f"Taxonomy Name not present for {field.name}" + ) + col_obj["taxonomy"] = ( + PolicyTagManagerClient() + .get_taxonomy(name=taxonomy_name) + .display_name + ) col_obj["policy_tags"] = ( PolicyTagManagerClient() - .get_policy_tag(name=field.policy_tags.names[0]) + .get_policy_tag(name=policy_tag_name) .display_name ) except Exception as exc: @@ -162,11 +175,11 @@ class BigquerySource(CommonDbSourceService): for tag in policy_tags: yield OMetaTagAndClassification( classification_request=CreateClassificationRequest( - name=self.service_connection.classificationName, + name=taxonomy.display_name, description="", ), tag_request=CreateTagRequest( - classification=self.service_connection.classificationName, + classification=taxonomy.display_name, name=tag.display_name, description="Bigquery Policy Tag", ), @@ -195,7 +208,7 @@ class BigquerySource(CommonDbSourceService): tagFQN=fqn.build( self.metadata, entity_type=Tag, - classification_name=self.service_connection.classificationName, + classification_name=column["taxonomy"], tag_name=column["policy_tags"], ), labelType="Automated", diff --git a/ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py b/ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py index 6c792be23b7..0c25270e3b0 100644 --- a/ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py +++ b/ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py @@ -50,7 +50,6 @@ from metadata.generated.schema.entity.services.pipelineService import PipelineSe from metadata.generated.schema.security.client.openMetadataJWTClientConfig import ( OpenMetadataJWTClientConfig, ) -from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.ometa.ometa_api import OpenMetadata OM_HOST_PORT = "http://localhost:8585/api" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json index 9fa0f18ccf2..4ed20309190 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json @@ -47,12 +47,6 @@ "description": "GCS Credentials", "$ref": "../../../../security/credentials/gcsCredentials.json" }, - "classificationName": { - "title": "Classification Name", - "description": "Custom OpenMetadata Classification name for BigQuery policy tags.", - "type": "string", - "default": "BigqueryPolicyTags" - }, "partitionQueryDuration": { "title": "Partition Query Duration", "description": "Duration for partitioning BigQuery tables.",