Status record Json encoding bug fixed and pandas not found fixed (#584)

ingestion/ingestion_dependency.sh

@@ -17,9 +17,4 @@
 #
 
 set -euo pipefail
-pip install --upgrade setuptools openmetadata-ingestion==0.2.1 apns
-# wget https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0-py3-none-any.whl
-# pip install en_core_web_sm-3.0.0-py3-none-any.whl
-python -m spacy download en_core_web_sm
-rm -rf en_core_web_sm-3.0.0-py3-none-any.whl
-pip install "simplescheduler@git+git://github.com/open-metadata/simplescheduler.git#egg=simplescheduler"
+pip install --upgrade setuptools 'openmetadata-ingestion[sample-data, elasticsearch, scheduler]'

ingestion/pipelines/sample_data.json

@@ -17,7 +17,7 @@
     }
   },
   "cron": {
-    "minute": "*/5",
+    "minute": "*/12",
     "hour": null,
     "day": null,
     "month": null,

ingestion/pipelines/sample_usage.json

@@ -33,7 +33,7 @@
     }
   },
   "cron": {
-    "minute": "*/5",
+    "minute": "*/6",
     "hour": null,
     "day": null,
     "month": null,

ingestion/pipelines/sample_users.json

@@ -18,10 +18,10 @@
     }
   },
   "cron": {
-    "minute": "*/5",
+    "minute": null,
     "hour": null,
-    "day": null,
-    "month": null,
-    "day_of_week": null
+    "day": "*/7",
+    "month": "*",
+    "day_of_week": "*"
   }
 }

ingestion/setup.py

@@ -108,7 +108,7 @@ plugins: Dict[str, Set[str]] = {
 build_options = {"includes": ["_cffi_backend"]}
 setup(
     name="openmetadata-ingestion",
-    version="0.3.0",
+    version="0.3.2",
     url="https://open-metadata.org/",
     author="OpenMetadata Committers",
     license="Apache License 2.0",

ingestion/src/metadata/ingestion/sink/elasticsearch.py

@@ -110,7 +110,10 @@ class ElasticsearchSink(Sink):
             dashboard_doc = self._create_dashboard_es_doc(record)
             self.elasticsearch_client.index(index=self.config.dashboard_index_name, id=str(dashboard_doc.dashboard_id),
                                             body=dashboard_doc.json())
-        self.status.records_written(record.name)
+        if (hasattr(record.name,'__root__')):
+            self.status.records_written(record.name.__root__)
+        else:
+            self.status.records_written(record.name)
 
     def _create_table_es_doc(self, table: Table):
         fqdn = table.fullyQualifiedName

ingestion/src/metadata/ingestion/sink/metadata_rest.py

@@ -125,7 +125,7 @@ class MetadataRestSink(Sink):
             created_topic = self.client.create_or_update_topic(topic)
             logger.info(
                 'Successfully ingested topic {}'.format(created_topic.name.__root__))
-            self.status.records_written(created_topic.name)
+            self.status.records_written(created_topic.name.__root__)
         except (APIError, ValidationError) as err:
             logger.error(
                 "Failed to ingest topic {} ".format(topic.name.__root__))