diff --git a/ingestion/ingestion_dependency.sh b/ingestion/ingestion_dependency.sh index 3fd059fa3e0..1e67378bf34 100755 --- a/ingestion/ingestion_dependency.sh +++ b/ingestion/ingestion_dependency.sh @@ -17,9 +17,4 @@ # set -euo pipefail -pip install --upgrade setuptools openmetadata-ingestion==0.2.1 apns -# wget https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0-py3-none-any.whl -# pip install en_core_web_sm-3.0.0-py3-none-any.whl -python -m spacy download en_core_web_sm -rm -rf en_core_web_sm-3.0.0-py3-none-any.whl -pip install "simplescheduler@git+git://github.com/open-metadata/simplescheduler.git#egg=simplescheduler" +pip install --upgrade setuptools 'openmetadata-ingestion[sample-data, elasticsearch, scheduler]' diff --git a/ingestion/pipelines/sample_data.json b/ingestion/pipelines/sample_data.json index 0243d737dbc..ffd3993b33a 100644 --- a/ingestion/pipelines/sample_data.json +++ b/ingestion/pipelines/sample_data.json @@ -17,7 +17,7 @@ } }, "cron": { - "minute": "*/5", + "minute": "*/12", "hour": null, "day": null, "month": null, diff --git a/ingestion/pipelines/sample_usage.json b/ingestion/pipelines/sample_usage.json index 4c591ea6442..fe5fc0af3ec 100644 --- a/ingestion/pipelines/sample_usage.json +++ b/ingestion/pipelines/sample_usage.json @@ -33,7 +33,7 @@ } }, "cron": { - "minute": "*/5", + "minute": "*/6", "hour": null, "day": null, "month": null, diff --git a/ingestion/pipelines/sample_users.json b/ingestion/pipelines/sample_users.json index b75eda2ead4..3fd692fe7f9 100644 --- a/ingestion/pipelines/sample_users.json +++ b/ingestion/pipelines/sample_users.json @@ -18,10 +18,10 @@ } }, "cron": { - "minute": "*/5", + "minute": null, "hour": null, - "day": null, - "month": null, - "day_of_week": null + "day": "*/7", + "month": "*", + "day_of_week": "*" } } diff --git a/ingestion/setup.py b/ingestion/setup.py index 49b7eef0e9f..5f5a80dafaa 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -108,7 +108,7 @@ plugins: Dict[str, Set[str]] = { build_options = {"includes": ["_cffi_backend"]} setup( name="openmetadata-ingestion", - version="0.3.0", + version="0.3.2", url="https://open-metadata.org/", author="OpenMetadata Committers", license="Apache License 2.0", diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch.py b/ingestion/src/metadata/ingestion/sink/elasticsearch.py index d041870b891..7efc6d8c739 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch.py @@ -110,7 +110,10 @@ class ElasticsearchSink(Sink): dashboard_doc = self._create_dashboard_es_doc(record) self.elasticsearch_client.index(index=self.config.dashboard_index_name, id=str(dashboard_doc.dashboard_id), body=dashboard_doc.json()) - self.status.records_written(record.name) + if (hasattr(record.name,'__root__')): + self.status.records_written(record.name.__root__) + else: + self.status.records_written(record.name) def _create_table_es_doc(self, table: Table): fqdn = table.fullyQualifiedName diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index 45c7aea1f08..0b19e06a571 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -125,7 +125,7 @@ class MetadataRestSink(Sink): created_topic = self.client.create_or_update_topic(topic) logger.info( 'Successfully ingested topic {}'.format(created_topic.name.__root__)) - self.status.records_written(created_topic.name) + self.status.records_written(created_topic.name.__root__) except (APIError, ValidationError) as err: logger.error( "Failed to ingest topic {} ".format(topic.name.__root__))