yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-21 14:59:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added Exception Handling for Bigquery Tags & Fix py-test CI (#7649)

Browse Source
This commit is contained in:
Mayur Singal 2022-09-22 14:08:04 +05:30 committed by GitHub
parent 569e8b9085
commit 680a186209
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/py-tests.yml vendored
View File

@ -27,7 +27,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
py-version: ['3.7', '3.8', '3.9', '3.10'] py-version: ['3.7', '3.8', '3.9']
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v2
@ -48,7 +48,7 @@ jobs:
- name: Install Ubuntu dependencies - name: Install Ubuntu dependencies
run: | run: |
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \ sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
librdkafka-dev unixodbc-dev libevent-dev unixodbc-dev libevent-dev python3-dev
- name: Generate models - name: Generate models
run: | run: |

37
ingestion/src/metadata/ingestion/source/database/bigquery.py
View File

@ -49,7 +49,6 @@ from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory
from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type
from metadata.ingestion.source.database.common_db_source import CommonDbSourceService from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
from metadata.utils import fqn from metadata.utils import fqn
from metadata.utils.filters import filter_by_table
from metadata.utils.logger import ingestion_logger from metadata.utils.logger import ingestion_logger
logger = ingestion_logger() logger = ingestion_logger()
@ -134,24 +133,28 @@ class BigquerySource(CommonDbSourceService):
:param _: :param _:
:return: :return:
""" """
taxonomies = PolicyTagManagerClient().list_taxonomies( try:
parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}" taxonomies = PolicyTagManagerClient().list_taxonomies(
) parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}"
for taxonomy in taxonomies:
policiy_tags = PolicyTagManagerClient().list_policy_tags(
parent=taxonomy.name
) )
for tag in policiy_tags: for taxonomy in taxonomies:
yield OMetaTagAndCategory( policiy_tags = PolicyTagManagerClient().list_policy_tags(
category_name=CreateTagCategoryRequest( parent=taxonomy.name
name=self.service_connection.tagCategoryName,
description="",
categoryType="Classification",
),
category_details=CreateTagRequest(
name=tag.display_name, description="Bigquery Policy Tag"
),
) )
for tag in policiy_tags:
yield OMetaTagAndCategory(
category_name=CreateTagCategoryRequest(
name=self.service_connection.tagCategoryName,
description="",
categoryType="Classification",
),
category_details=CreateTagRequest(
name=tag.display_name, description="Bigquery Policy Tag"
),
)
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Skipping Policy Tag: {exc}")
def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]: def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]:
""" """