yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-07-22 08:50:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

added data products index and missing code of stored procedure index (#13175)

Browse Source
This commit is contained in:
07Himank 2023-09-13 18:52:49 +05:30 committed by GitHub
parent c5edf37cfe
commit 0059645b64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 1091 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java
View File

@ -43,6 +43,7 @@ public class DataProductRepository extends EntityRepository<DataProduct> {
dao,
UPDATE_FIELDS,
UPDATE_FIELDS);
supportsSearchIndex = true;
}
@Override

1
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/StoredProcedureRepository.java
View File

@ -27,6 +27,7 @@ public class StoredProcedureRepository extends EntityRepository<StoredProcedure>
dao,
PATCH_FIELDS,
UPDATE_FIELDS);
supportsSearchIndex = true;
}
@Override

4
openmetadata-service/src/main/java/org/openmetadata/service/search/IndexUtil.java
View File

@ -144,6 +144,10 @@ public class IndexUtil {
return SearchIndexDefinition.ElasticSearchIndexType.SEARCH_SERVICE_SEARCH_INDEX;
} else if (type.equalsIgnoreCase(Entity.DOMAIN)) {
return SearchIndexDefinition.ElasticSearchIndexType.DOMAIN_SEARCH_INDEX;
} else if (type.equalsIgnoreCase(Entity.STORED_PROCEDURE)) {
return SearchIndexDefinition.ElasticSearchIndexType.STORED_PROCEDURE_SEARCH_INDEX;
} else if (type.equalsIgnoreCase(Entity.DATA_PRODUCT)) {
return SearchIndexDefinition.ElasticSearchIndexType.DATA_PRODUCTS_SEARCH_INDEX;
}
throw new EventPublisherException("Failed to find index doc for type " + type);
}

6
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexDefinition.java
View File

@ -57,6 +57,12 @@ public class SearchIndexDefinition {
Entity.TEST_SUITE, "test_suite_search_index", "/elasticsearch/%s/test_suite_index_mapping.json"),
CHART_INDEX_SEARCH_INDEX(Entity.CHART, "chart_search_index", "/elasticsearch/%s/chart_index_mapping.json"),
STORED_PROCEDURE_SEARCH_INDEX(
Entity.STORED_PROCEDURE,
"stored_procedure_search_index",
"/elasticsearch/%s/stored_procedure_index_mapping.json"),
DATA_PRODUCTS_SEARCH_INDEX(
Entity.DATA_PRODUCT, "data_products_search_index", "/elasticsearch/%s/data_products_index_mapping.json"),
DASHBOARD_DATA_MODEL_SEARCH_INDEX(
Entity.DASHBOARD_DATA_MODEL,

8
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java
View File

@ -13,8 +13,10 @@ import org.openmetadata.schema.entity.data.GlossaryTerm;
import org.openmetadata.schema.entity.data.MlModel;
import org.openmetadata.schema.entity.data.Pipeline;
import org.openmetadata.schema.entity.data.Query;
import org.openmetadata.schema.entity.data.StoredProcedure;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.data.Topic;
import org.openmetadata.schema.entity.domains.DataProduct;
import org.openmetadata.schema.entity.domains.Domain;
import org.openmetadata.schema.entity.services.DashboardService;
import org.openmetadata.schema.entity.services.DatabaseService;
@ -33,6 +35,7 @@ import org.openmetadata.service.search.indexes.ContainerIndex;
import org.openmetadata.service.search.indexes.DashboardDataModelIndex;
import org.openmetadata.service.search.indexes.DashboardIndex;
import org.openmetadata.service.search.indexes.DashboardServiceIndex;
import org.openmetadata.service.search.indexes.DataProductIndex;
import org.openmetadata.service.search.indexes.DatabaseIndex;
import org.openmetadata.service.search.indexes.DatabaseSchemaIndex;
import org.openmetadata.service.search.indexes.DatabaseServiceIndex;
@ -47,6 +50,7 @@ import org.openmetadata.service.search.indexes.PipelineServiceIndex;
import org.openmetadata.service.search.indexes.QueryIndex;
import org.openmetadata.service.search.indexes.SearchIndex;
import org.openmetadata.service.search.indexes.SearchServiceIndex;
import org.openmetadata.service.search.indexes.StoredProcedureIndex;
import org.openmetadata.service.search.indexes.TableIndex;
import org.openmetadata.service.search.indexes.TagIndex;
import org.openmetadata.service.search.indexes.TeamIndex;
@ -113,6 +117,10 @@ public class SearchIndexFactory {
return new PipelineServiceIndex((PipelineService) entity);
case Entity.DOMAIN:
return new DomainIndex((Domain) entity);
case Entity.STORED_PROCEDURE:
return new StoredProcedureIndex((StoredProcedure) entity);
case Entity.DATA_PRODUCT:
return new DataProductIndex((DataProduct) entity);
default:
LOG.warn("Ignoring Entity Type {}", entityType);
}

24
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java Normal file
View File

@ -0,0 +1,24 @@
package org.openmetadata.service.search.indexes;
import java.util.List;
import java.util.Map;
import org.openmetadata.schema.entity.domains.DataProduct;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.util.JsonUtils;
public class DataProductIndex implements ElasticSearchIndex {
private static final List<String> excludeFields = List.of("changeDescription");
final DataProduct dataProduct;
public DataProductIndex(DataProduct dataProduct) {
this.dataProduct = dataProduct;
}
public Map<String, Object> buildESDoc() {
Map<String, Object> doc = JsonUtils.getMap(dataProduct);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
return doc;
}
}

24
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java Normal file
View File

@ -0,0 +1,24 @@
package org.openmetadata.service.search.indexes;
import java.util.List;
import java.util.Map;
import org.openmetadata.schema.entity.data.StoredProcedure;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.util.JsonUtils;
public class StoredProcedureIndex implements ElasticSearchIndex {
private static final List<String> excludeFields = List.of("changeDescription");
final StoredProcedure storedProcedure;
public StoredProcedureIndex(StoredProcedure storedProcedure) {
this.storedProcedure = storedProcedure;
}
public Map<String, Object> buildESDoc() {
Map<String, Object> doc = JsonUtils.getMap(storedProcedure);
SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
return doc;
}
}

45
openmetadata-service/src/main/resources/elasticsearch/en/chart_Index_mapping.json
View File

@ -102,6 +102,51 @@
"type": "text"
},
"owner": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"normalizer": "lowercase_normalizer",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "text"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"followers": {
"properties": {

220
openmetadata-service/src/main/resources/elasticsearch/en/data_products_index_mapping.json Normal file
View File

@ -0,0 +1,220 @@
{
"settings": {
"analysis": {
"normalizer": {
"lowercase_normalizer": {
"type": "custom",
"char_filter": [],
"filter": [
"lowercase"
]
}
},
"analyzer": {
"om_analyzer": {
"tokenizer": "letter",
"filter": [
"lowercase",
"om_stemmer"
]
},
"om_ngram": {
"tokenizer": "ngram",
"min_gram": 1,
"max_gram": 2,
"filter": [
"lowercase"
]
}
},
"filter": {
"om_stemmer": {
"type": "stemmer",
"name": "english"
}
}
}
},
"mappings": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text",
"analyzer": "om_analyzer",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"ngram": {
"type": "text",
"analyzer": "om_ngram"
}
}
},
"fullyQualifiedName": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"displayName": {
"type": "text",
"analyzer": "om_analyzer",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"ngram": {
"type": "text",
"analyzer": "om_ngram"
}
}
},
"description": {
"type": "text",
"analyzer": "om_analyzer",
"fields": {
"ngram": {
"type": "text",
"analyzer": "om_ngram"
}
}
},
"version": {
"type": "float"
},
"updatedAt": {
"type": "date",
"format": "epoch_second"
},
"updatedBy": {
"type": "text"
},
"href": {
"type": "text"
},
"owner": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "text"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"domain": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"assets": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
}
}
}
}

114
openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json
View File

@ -83,7 +83,14 @@
}
},
"storedProcedureCode": {
"type": "keyword"
"properties": {
"language": {
"type": "text"
},
"code": {
"type": "text"
}
}
},
"version": {
"type": "float"
@ -240,9 +247,7 @@
"type": "keyword"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
"type": "text"
},
"owner": {
"properties": {
@ -294,7 +299,106 @@
"type": "keyword"
},
"votes" : {
"type": "keyword"
"properties": {
"upVotes": {
"type": "long"
},
"downVotes": {
"type": "long"
},
"upVoters": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"downVoters": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
}
}
},
"code": {
"type": "keyword"

230
openmetadata-service/src/main/resources/elasticsearch/jp/data_products_index_mapping.json Normal file
View File

@ -0,0 +1,230 @@
{
"settings": {
"analysis": {
"normalizer": {
"lowercase_normalizer": {
"type": "custom",
"char_filter": [],
"filter": [
"lowercase"
]
}
},
"analyzer": {
"om_analyzer": {
"tokenizer": "letter",
"filter": [
"lowercase",
"om_stemmer"
]
},
"om_analyzer_jp" : {
"tokenizer" : "kuromoji_tokenizer",
"type" : "custom",
"filter" : [
"kuromoji_baseform",
"kuromoji_part_of_speech",
"kuromoji_number",
"kuromoji_stemmer"
]
},
"om_ngram": {
"tokenizer": "ngram",
"min_gram": 1,
"max_gram": 2,
"filter": [
"lowercase"
]
}
},
"filter": {
"om_stemmer": {
"type": "stemmer",
"name": "english"
}
}
}
},
"mappings": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text",
"analyzer": "om_analyzer_jp",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"ngram": {
"type": "text",
"analyzer": "om_ngram"
}
}
},
"fullyQualifiedName": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"displayName": {
"type": "text",
"analyzer": "om_analyzer_jp",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"ngram": {
"type": "text",
"analyzer": "om_ngram"
}
}
},
"description": {
"type": "text",
"analyzer": "om_analyzer_jp",
"fields": {
"ngram": {
"type": "text",
"analyzer": "om_ngram"
}
}
},
"version": {
"type": "float"
},
"updatedAt": {
"type": "date",
"format": "epoch_second"
},
"updatedBy": {
"type": "text"
},
"href": {
"type": "text"
},
"owner": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "text"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"domain": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"assets": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
}
}
}
}

114
openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json
View File

@ -86,7 +86,14 @@
}
},
"storedProcedureCode": {
"type": "keyword"
"properties": {
"language": {
"type": "text"
},
"code": {
"type": "text"
}
}
},
"version": {
"type": "float"
@ -243,9 +250,7 @@
"type": "keyword"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
"type": "text"
},
"owner": {
"properties": {
@ -297,7 +302,106 @@
"type": "keyword"
},
"votes" : {
"type": "keyword"
"properties": {
"upVotes": {
"type": "long"
},
"downVotes": {
"type": "long"
},
"upVoters": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"downVoters": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
}
}
},
"code": {
"type": "keyword"

201
openmetadata-service/src/main/resources/elasticsearch/zh/data_products_index_mapping.json Normal file
View File

@ -0,0 +1,201 @@
{
"settings": {
"analysis": {
"normalizer": {
"lowercase_normalizer": {
"type": "custom",
"char_filter": [],
"filter": [
"lowercase"
]
}
},
"analyzer": {
"om_analyzer": {
"tokenizer": "letter",
"filter": [
"lowercase",
"om_stemmer"
]
}
},
"filter": {
"om_stemmer": {
"type": "stemmer",
"name": "english"
}
}
}
},
"mappings": {
"properties": {
"id": {
"type": "text"
},
"name": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"displayName": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"description": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"version": {
"type": "float"
},
"updatedAt": {
"type": "date",
"format": "epoch_second"
},
"updatedBy": {
"type": "text"
},
"href": {
"type": "text"
},
"owner": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "text"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"domain": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"assets": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
}
}
}
}

114
openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json
View File

@ -64,7 +64,14 @@
"search_analyzer": "ik_smart"
},
"storedProcedureCode": {
"type": "keyword"
"properties": {
"language": {
"type": "text"
},
"code": {
"type": "text"
}
}
},
"version": {
"type": "float"
@ -221,9 +228,7 @@
"type": "keyword"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
"type": "text"
},
"owner": {
"properties": {
@ -275,7 +280,106 @@
"type": "keyword"
},
"votes" : {
"type": "keyword"
"properties": {
"upVotes": {
"type": "long"
},
"downVotes": {
"type": "long"
},
"upVoters": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
},
"downVoters": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 36
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"displayName": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "text"
},
"href": {
"type": "text"
}
}
}
}
},
"code": {
"type": "keyword"