yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-25 14:38:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ingestion - fix pytest workflow errors and modify airflow pipeline (#4178)

Browse Source
This commit is contained in:
Ayush Shah 2022-04-18 11:36:21 +05:30 committed by GitHub
parent d179692ad3
commit 4f5c5a9e23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
ingestion/examples/airflow/dags/airflow_metadata_to_es.py
View File

@ -35,12 +35,18 @@ config = """
{
"source": {
"type": "metadata",
"config": {
"include_tables": "true",
"include_topics": "true",
"include_dashboards": "true",
"limit_records": 10
}
"serviceName": "openMetadata",
"serviceConnection": {
"config":{
"type":"MetadataES",
"includeTables": "true",
"includeUsers": "true",
"includeTopics": "true",
"includeDashboards": "true",
"limitRecords": 10
}
},
"sourceConfig":{"config":{}}
},
"sink": {
"type": "elasticsearch",
@ -52,11 +58,10 @@ config = """
"es_port": 9200
}
},
"metadata_server": {
"type": "metadata-server",
"config": {
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "no-auth"
}
}
}

2
ingestion/setup.py
View File

@ -46,7 +46,7 @@ base_requirements = {
"PyYAML",
"jsonschema",
"sqllineage==1.3.3",
"MarkupSafe>=2.0",
"MarkupSafe==2.0.1",
}
report_requirements = {