yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-01 21:23:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #12082 - Bump PyAthena version (#13464)

Browse Source
This commit is contained in:
Pere Miquel Brull 2023-10-09 20:47:19 +02:00 committed by GitHub
parent 8b028a737b
commit f6a87ee02a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 20 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
ingestion/examples/airflow/dags/airflow_metadata_to_es.py
View File

@ -37,32 +37,23 @@ config = """
{
"source": {
"type": "metadata_elasticsearch",
"serviceName": "openMetadata",
"serviceName": "Openmetadata",
"serviceConnection": {
"config":{
"type":"MetadataES",
"includeTables": "true",
"includeUsers": "true",
"includeTopics": "true",
"includeDashboards": "true",
"limitRecords": 10
}
"type":"MetadataES"
}
},
"sourceConfig":{"config":{}}
"sourceConfig":{"config":{
"type": "MetadataToElasticSearch"
}}
},
"sink": {
"type": "elasticsearch",
"config": {
"index_tables": "true",
"index_topics": "true",
"index_dashboards": "true",
"es_host": "elasticsearch",
"es_port": 9200
}
"type": "metadata-rest",
"config": {}
},
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://openmetadata-server:8585/api",
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig":{
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

2
ingestion/setup.py
View File

@ -133,7 +133,7 @@ base_requirements = {
plugins: Dict[str, Set[str]] = {
"airflow": {VERSIONS["airflow"]}, # Same as ingestion container. For development.
"amundsen": {VERSIONS["neo4j"]},
"athena": {"pyathena==2.25.2"},
"athena": {"pyathena==3.0.8"},
"atlas": {},
"azuresql": {VERSIONS["pyodbc"]},
"azure-sso": {VERSIONS["msal"]},

2
ingestion/src/metadata/ingestion/source/database/athena/metadata.py
View File

@ -13,7 +13,7 @@
from typing import Iterable
from pyathena.sqlalchemy_athena import AthenaDialect
from pyathena.sqlalchemy.base import AthenaDialect
from sqlalchemy import types
from sqlalchemy.engine import reflection

10
ingestion/src/metadata/pii/processor.py
View File

@ -62,9 +62,17 @@ class PIIProcessor(Processor):
DatabaseServiceProfilerPipeline, self.config.source.sourceConfig.config
) # Used to satisfy type checked
self.ner_scanner = NERScanner()
self._ner_scanner = None
self.confidence_threshold = self.source_config.confidence
@property
def ner_scanner(self) -> NERScanner:
"""Load the NER Scanner only if called"""
if self._ner_scanner is None:
self._ner_scanner = NERScanner()
return self._ner_scanner
@classmethod
def create(cls, config_dict: dict, metadata: OpenMetadata) -> "Step":
config = parse_workflow_config_gracefully(config_dict)