yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-06 04:26:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #515: Ingestion: Add ES configuration to allow port (#516)

Browse Source
This commit is contained in:
Sriharsha Chintalapani 2021-09-17 08:57:41 -07:00 committed by GitHub
parent cc6a4de72f
commit 4c6c8fd446
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ingestion/pipelines/metadata_to_es.json
View File

@ -13,7 +13,8 @@
"index_tables": "true",
"index_topics": "true",
"index_dashboards": "true",
"es_host_port": "localhost"
"es_host": "localhost",
"es_port": 9300
}
},
"metadata_server": {

4
ingestion/setup.py
View File

@ -42,8 +42,7 @@ scheduler_requirements = {
}
profiler_requirements = {
"openmetadata-data-profiler@git+git://github.com/open-metadata/data-profiler.git#egg"
"=openmetadata-data-profiler"
"openmetadata-data-profiler@git+git://github.com/open-metadata/data-profiler.git#egg=openmetadata-data-profiler"
}
base_requirements = {
@ -92,7 +91,6 @@ plugins: Dict[str, Set[str]] = {
"pii-processor": {"pandas~=1.3.1"},
"presto": {"pyhive~=0.6.3"},
"postgres": {"pymysql>=1.0.2", "psycopg2-binary", "GeoAlchemy2"},
"profiler": {"ruamel.yaml", "jsonpatch", "pandas", "IPython", "jsonschema", "scipy", "mistune", "altair", "tzlocal"},
"redshift": {"sqlalchemy-redshift", "GeoAlchemy2", "psycopg2-binary"},
"redshift-usage": {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
"scheduler": scheduler_requirements,

6
ingestion/src/metadata/ingestion/sink/elasticsearch.py
View File

@ -37,7 +37,8 @@ logger = logging.getLogger(__name__)
class ElasticSearchConfig(ConfigModel):
es_host_port: str
es_host: str
es_port: int = 9200
index_tables: Optional[bool] = True
index_topics: Optional[bool] = False
index_dashboards: Optional[bool] = False
@ -68,7 +69,8 @@ class ElasticsearchSink(Sink):
self.rest = OpenMetadataAPIClient(self.metadata_config)
self.elasticsearch_doc_type = '_doc'
self.elasticsearch_client = Elasticsearch([
{'host': self.config.es_host_port},
{'host': self.config.es_host,
'port': self.config.es_port},
])
if self.config.index_tables:
self._check_or_create_index(self.config.table_index_name, TABLE_ELASTICSEARCH_INDEX_MAPPING)