yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-17 10:35:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #3865: Service Connection & Schema Name - Fix ES indexes

Browse Source
This commit is contained in:
Sriharsha Chintalapani 2022-04-10 09:47:36 -07:00
parent 42dcb383c0
commit 4a572713a7
8 changed files with 30 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java
View File

@ -272,6 +272,9 @@ class TableESIndex extends ElasticSearchIndex {
String database; String database;
@JsonProperty("database_schema")
String databaseSchema;
@JsonProperty("table_type") @JsonProperty("table_type")
String tableType; String tableType;
@ -348,14 +351,11 @@ class TableESIndex extends ElasticSearchIndex {
.tier(parseTags.tierTag); .tier(parseTags.tierTag);
if (table.getDatabase() != null) { if (table.getDatabase() != null) {
String databaseFQN = table.getDatabase().getName(); tableESIndexBuilder.database(table.getDatabase().getName());
// TODO fix this code
String[] databaseFQNSplit = FullyQualifiedName.split(databaseFQN);
if (databaseFQNSplit.length == 2) {
tableESIndexBuilder.database(databaseFQNSplit[1]);
} else {
tableESIndexBuilder.database(databaseFQNSplit[0]);
} }
if (table.getDatabaseSchema() != null) {
tableESIndexBuilder.databaseSchema(table.getDatabaseSchema().getName());
} }
if (table.getService() != null) { if (table.getService() != null) {

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/search/SearchResource.java
View File

@ -245,8 +245,8 @@ public class SearchResource {
hb.field(highlightColumns); hb.field(highlightColumns);
hb.field(highlightColumnDescriptions); hb.field(highlightColumnDescriptions);
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
searchSourceBuilder.aggregation(AggregationBuilders.terms("Database").field("database")).highlighter(hb); searchSourceBuilder.aggregation(AggregationBuilders.terms("Database").field("database"));
searchSourceBuilder.aggregation(AggregationBuilders.terms("DatabaseSchema").field("database_schema"));
return addAggregation(searchSourceBuilder); return addAggregation(searchSourceBuilder);
} }
@ -330,6 +330,7 @@ public class SearchResource {
private SearchSourceBuilder addAggregation(SearchSourceBuilder builder) { private SearchSourceBuilder addAggregation(SearchSourceBuilder builder) {
builder builder
.aggregation(AggregationBuilders.terms("Service").field("service_type").size(MAX_AGGREGATE_SIZE)) .aggregation(AggregationBuilders.terms("Service").field("service_type").size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms("ServiceName").field("service").size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms("ServiceCategory").field("service_category").size(MAX_AGGREGATE_SIZE)) .aggregation(AggregationBuilders.terms("ServiceCategory").field("service_category").size(MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms("EntityType").field("entity_type")) .aggregation(AggregationBuilders.terms("EntityType").field("entity_type"))
.aggregation(AggregationBuilders.terms("Tier").field("tier")) .aggregation(AggregationBuilders.terms("Tier").field("tier"))

3
catalog-rest-service/src/main/resources/elasticsearch/table_index_mapping.json
View File

@ -52,6 +52,9 @@
"database": { "database": {
"type": "keyword" "type": "keyword"
}, },
"database_schema": {
"type": "keyword"
},
"suggest": { "suggest": {
"type": "completion" "type": "completion"
}, },

10
ingestion/examples/sample_data/dashboards/service.json
View File

@ -1,12 +1,12 @@
{ {
"name": "sample_superset", "name": "sample_superset",
"serviceType": "Superset", "serviceType": "Superset",
"description": "Supset Service", "description": "Superset Service",
"connection": { "dashboardConnection": {
"config": { "config": {
"supersetURL": "http://localhost:8088" "supersetURL": "http://localhost:8088",
}
},
"username": "admin", "username": "admin",
"password": "admin" "password": "admin"
}
}
} }

10
ingestion/pipelines/metadata_to_es.json
View File

@ -1,6 +1,7 @@
{ {
"source": { "source": {
"type": "metadata", "type": "metadata",
"serviceName": "openMetadata",
"config": { "config": {
"include_tables": "true", "include_tables": "true",
"include_topics": "true", "include_topics": "true",
@ -18,11 +19,10 @@
"es_port": 9200 "es_port": 9200
} }
}, },
"metadata_server": { "workflowConfig": {
"type": "metadata-server", "openMetadataServerConfig": {
"config": { "hostPort": "http://localhost:8585/api",
"api_endpoint": "http://localhost:8585/api", "authProvider": "no-auth"
"auth_provider_type": "no-auth"
} }
} }
} }

1
ingestion/src/metadata/ingestion/models/table_metadata.py
View File

@ -56,6 +56,7 @@ class TableESDocument(BaseModel):
table_id: str table_id: str
deleted: bool deleted: bool
database: str database: str
database_schema: str
service: str service: str
service_type: str service_type: str
service_category: str service_category: str

3
ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py
View File

@ -68,6 +68,9 @@ TABLE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"database": { "database": {
"type": "keyword" "type": "keyword"
}, },
"database_schema": {
"type": "keyword"
},
"suggest": { "suggest": {
"type": "completion" "type": "completion"
}, },

1
ingestion/src/metadata/utils/helpers.py
View File

@ -154,6 +154,7 @@ def get_dashboard_service_or_create(
return service return service
else: else:
dashboard_config = {"config": config} dashboard_config = {"config": config}
print(dashboard_config)
created_service = metadata.create_or_update( created_service = metadata.create_or_update(
CreateDashboardServiceRequest( CreateDashboardServiceRequest(
name=service_name, name=service_name,