yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-06-27 04:22:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Status record Json encoding bug fixed and pandas not found fixed (#584)

Browse Source
This commit is contained in:
Ayush Shah 2021-09-26 02:24:04 +05:30 committed by GitHub
parent 9d24d62471
commit 627481f181
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ingestion/ingestion_dependency.sh
View File

@ -17,9 +17,4 @@
# #
set -euo pipefail set -euo pipefail
pip install --upgrade setuptools openmetadata-ingestion==0.2.1 apns pip install --upgrade setuptools 'openmetadata-ingestion[sample-data, elasticsearch, scheduler]'
# wget https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0-py3-none-any.whl
# pip install en_core_web_sm-3.0.0-py3-none-any.whl
python -m spacy download en_core_web_sm
rm -rf en_core_web_sm-3.0.0-py3-none-any.whl
pip install "simplescheduler@git+git://github.com/open-metadata/simplescheduler.git#egg=simplescheduler"

2
ingestion/pipelines/sample_data.json
View File

@ -17,7 +17,7 @@
} }
}, },
"cron": { "cron": {
"minute": "*/5", "minute": "*/12",
"hour": null, "hour": null,
"day": null, "day": null,
"month": null, "month": null,

2
ingestion/pipelines/sample_usage.json
View File

@ -33,7 +33,7 @@
} }
}, },
"cron": { "cron": {
"minute": "*/5", "minute": "*/6",
"hour": null, "hour": null,
"day": null, "day": null,
"month": null, "month": null,

8
ingestion/pipelines/sample_users.json
View File

@ -18,10 +18,10 @@
} }
}, },
"cron": { "cron": {
"minute": "*/5", "minute": null,
"hour": null, "hour": null,
"day": null, "day": "*/7",
"month": null, "month": "*",
"day_of_week": null "day_of_week": "*"
} }
} }

2
ingestion/setup.py
View File

@ -108,7 +108,7 @@ plugins: Dict[str, Set[str]] = {
build_options = {"includes": ["_cffi_backend"]} build_options = {"includes": ["_cffi_backend"]}
setup( setup(
name="openmetadata-ingestion", name="openmetadata-ingestion",
version="0.3.0", version="0.3.2",
url="https://open-metadata.org/", url="https://open-metadata.org/",
author="OpenMetadata Committers", author="OpenMetadata Committers",
license="Apache License 2.0", license="Apache License 2.0",

5
ingestion/src/metadata/ingestion/sink/elasticsearch.py
View File

@ -110,7 +110,10 @@ class ElasticsearchSink(Sink):
dashboard_doc = self._create_dashboard_es_doc(record) dashboard_doc = self._create_dashboard_es_doc(record)
self.elasticsearch_client.index(index=self.config.dashboard_index_name, id=str(dashboard_doc.dashboard_id), self.elasticsearch_client.index(index=self.config.dashboard_index_name, id=str(dashboard_doc.dashboard_id),
body=dashboard_doc.json()) body=dashboard_doc.json())
self.status.records_written(record.name) if (hasattr(record.name,'__root__')):
self.status.records_written(record.name.__root__)
else:
self.status.records_written(record.name)
def _create_table_es_doc(self, table: Table): def _create_table_es_doc(self, table: Table):
fqdn = table.fullyQualifiedName fqdn = table.fullyQualifiedName

2
ingestion/src/metadata/ingestion/sink/metadata_rest.py
View File

@ -125,7 +125,7 @@ class MetadataRestSink(Sink):
created_topic = self.client.create_or_update_topic(topic) created_topic = self.client.create_or_update_topic(topic)
logger.info( logger.info(
'Successfully ingested topic {}'.format(created_topic.name.__root__)) 'Successfully ingested topic {}'.format(created_topic.name.__root__))
self.status.records_written(created_topic.name) self.status.records_written(created_topic.name.__root__)
except (APIError, ValidationError) as err: except (APIError, ValidationError) as err:
logger.error( logger.error(
"Failed to ingest topic {} ".format(topic.name.__root__)) "Failed to ingest topic {} ".format(topic.name.__root__))