From 4f5c5a9e235d26a5313f2d4c2e148f3f4783a9b3 Mon Sep 17 00:00:00 2001
From: Ayush Shah <ayush@getcollate.io>
Date: Mon, 18 Apr 2022 11:36:21 +0530
Subject: [PATCH] Ingestion - fix pytest workflow errors and modify airflow
 pipeline (#4178)

---
 .../airflow/dags/airflow_metadata_to_es.py    | 27 +++++++++++--------
 ingestion/setup.py                            |  2 +-
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/ingestion/examples/airflow/dags/airflow_metadata_to_es.py b/ingestion/examples/airflow/dags/airflow_metadata_to_es.py
index d565e90c983..2e7473fad3f 100644
--- a/ingestion/examples/airflow/dags/airflow_metadata_to_es.py
+++ b/ingestion/examples/airflow/dags/airflow_metadata_to_es.py
@@ -35,12 +35,18 @@ config = """
 {
   "source": {
     "type": "metadata",
-    "config": {
-      "include_tables": "true",
-      "include_topics": "true",
-      "include_dashboards": "true",
-      "limit_records": 10
-    }
+    "serviceName": "openMetadata",
+    "serviceConnection": {
+      "config":{
+        "type":"MetadataES",
+        "includeTables": "true",
+        "includeUsers": "true",
+        "includeTopics": "true",
+        "includeDashboards": "true",
+        "limitRecords": 10
+      } 
+    },
+    "sourceConfig":{"config":{}}
   },
   "sink": {
     "type": "elasticsearch",
@@ -52,11 +58,10 @@ config = """
       "es_port": 9200
     }
   },
-  "metadata_server": {
-    "type": "metadata-server",
-    "config": {
-      "api_endpoint": "http://localhost:8585/api",
-      "auth_provider_type": "no-auth"
+  "workflowConfig": {
+    "openMetadataServerConfig": {
+      "hostPort": "http://localhost:8585/api",
+      "authProvider": "no-auth"
     }
   }
 }
diff --git a/ingestion/setup.py b/ingestion/setup.py
index 864f327214e..e6a5c3b5205 100644
--- a/ingestion/setup.py
+++ b/ingestion/setup.py
@@ -46,7 +46,7 @@ base_requirements = {
     "PyYAML",
     "jsonschema",
     "sqllineage==1.3.3",
-    "MarkupSafe>=2.0",
+    "MarkupSafe==2.0.1",
 }
 
 report_requirements = {