yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-22 06:14:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #12779: Add support for SearchIndexes for ElasticSearch and OpenSearch (#12782)

Browse Source 
* Fix #12779: Add support for SearchIndexes for ElasticSearch and OpenSearch

* Fix #12779: Add support for SearchIndexes for ElasticSearch and OpenSearch

* Fix #12779: Add support for SearchIndexes for ElasticSearch and OpenSearch

* Rebase fixes with main

* Add Sample Data

* lint fix

* remove unused import

* Fix service count test

---------

Co-authored-by: ulixius9 <mayursingal9@gmail.com>
This commit is contained in:
Sriharsha Chintalapani 2023-08-10 16:47:37 -07:00 committed by GitHub
parent 60ff64c8ed
commit 7aaf654f01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 3129 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql
View File

@ -1,27 +1,3 @@
-- create domain entity table
CREATE TABLE IF NOT EXISTS domain_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);
-- create data product entity table
CREATE TABLE IF NOT EXISTS data_product_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);
-- Rename includeTempTables with includeTransTables
UPDATE dbservice_entity
SET json = JSON_REMOVE(

51
bootstrap/sql/com.mysql.cj.jdbc.Driver/v015__create_db_connection_info.sql
View File

@ -1,3 +1,54 @@
-- column deleted not needed for entities that don't support soft delete
ALTER TABLE query_entity DROP COLUMN deleted;
ALTER TABLE event_subscription_entity DROP COLUMN deleted;
-- create domain entity table
CREATE TABLE IF NOT EXISTS domain_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);
-- create data product entity table
CREATE TABLE IF NOT EXISTS data_product_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);
-- create search service entity
CREATE TABLE IF NOT EXISTS search_service_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
nameHash VARCHAR(256) NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE (nameHash)
);
-- create search index entity
CREATE TABLE IF NOT EXISTS search_index_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSON NOT NULL,
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
PRIMARY KEY (id),
UNIQUE (fqnHash)
);

26
bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql
View File

@ -1,28 +1,4 @@
-- create domain entity table
CREATE TABLE IF NOT EXISTS domain_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);
-- create data product entity table
CREATE TABLE IF NOT EXISTS data_product_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);
-- Rename includeTempTables in snowflake to includeTransientTables
-- Rename includeTempTables in snowflake to includeTransientTables
UPDATE dbservice_entity
SET json = jsonb_set(json::jsonb #- '{connection,config,includeTempTables}', '{connection,config,includeTransientTables}',

51
bootstrap/sql/org.postgresql.Driver/v015__create_db_connection_info.sql
View File

@ -1,3 +1,54 @@
-- column deleted not needed for entities that don't support soft delete
ALTER TABLE query_entity DROP COLUMN deleted;
ALTER TABLE event_subscription_entity DROP COLUMN deleted;
-- create domain entity table
CREATE TABLE IF NOT EXISTS domain_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);
-- create data product entity table
CREATE TABLE IF NOT EXISTS data_product_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);
-- create search service entity
CREATE TABLE IF NOT EXISTS search_service_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
nameHash VARCHAR(256) NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (nameHash)
);
-- create search index entity
CREATE TABLE IF NOT EXISTS search_index_entity (
id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL,
name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL,
fqnHash VARCHAR(256) NOT NULL,
json JSONB NOT NULL,
updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL,
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL,
deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED,
PRIMARY KEY (id),
UNIQUE (fqnHash)
);

76
ingestion/examples/sample_data/searchIndexes/searchIndexes.json Normal file
View File

@ -0,0 +1,76 @@
{
"searchIndexes": [
{
"id": "e093dd27-390e-4360-8efd-e4d63ec167a9",
"name": "table_search_index",
"displayName": "TableSearchIndex",
"fullyQualifiedName": "elasticsearch_sample.table_search_index",
"description": "Table Search Index",
"version": 0.1,
"updatedAt": 1638354087591,
"serviceType": "ElasticSearch",
"fields": [
{
"name": "name",
"dataType": "TEXT",
"dataTypeDisplay": "text",
"description": "Table Entity Name.",
"tags": []
},
{
"name": "displayName",
"dataType": "TEXT",
"dataTypeDisplay": "text",
"description": "Table Entity DisplayName.",
"tags": []
},
{
"name": "description",
"dataType": "TEXT",
"dataTypeDisplay": "text",
"description": "Table Entity Description.",
"tags": []
},
{
"name": "columns",
"dataType": "NESTED",
"dataTypeDisplay": "nested",
"description": "Table Columns.",
"tags": [],
"children": [
{
"name": "name",
"dataType": "TEXT",
"dataTypeDisplay": "text",
"description": "Column Name.",
"tags": []
},
{
"name": "displayName",
"dataType": "TEXT",
"dataTypeDisplay": "text",
"description": "Column DisplayName.",
"tags": []
},
{
"name": "description",
"dataType": "TEXT",
"dataTypeDisplay": "text",
"description": "Column Description.",
"tags": []
}
]
},
{
"name": "databaseSchema",
"dataType": "TEXT",
"dataTypeDisplay": "text",
"description": "Database Schema that this table belongs to.",
"tags": []
}
],
"tags": [],
"followers": []
}
]
}

12
ingestion/examples/sample_data/searchIndexes/service.json Normal file
View File

@ -0,0 +1,12 @@
{
"type": "elasticsearch",
"serviceName": "elasticsearch_sample",
"serviceConnection": {
"config": {
"type": "ElasticSearch",
"hostPort": "localhost:9200"
}
},
"sourceConfig": {
}
}

10
ingestion/src/metadata/ingestion/api/parser.py
View File

@ -42,6 +42,10 @@ from metadata.generated.schema.entity.services.pipelineService import (
PipelineConnection,
PipelineServiceType,
)
from metadata.generated.schema.entity.services.searchService import (
SearchConnection,
SearchServiceType,
)
from metadata.generated.schema.entity.services.storageService import (
StorageConnection,
StorageServiceType,
@ -74,6 +78,10 @@ from metadata.generated.schema.metadataIngestion.pipelineServiceMetadataPipeline
PipelineMetadataConfigType,
PipelineServiceMetadataPipeline,
)
from metadata.generated.schema.metadataIngestion.searchServiceMetadataPipeline import (
SearchMetadataConfigType,
SearchServiceMetadataPipeline,
)
from metadata.generated.schema.metadataIngestion.storageServiceMetadataPipeline import (
StorageMetadataConfigType,
StorageServiceMetadataPipeline,
@ -102,6 +110,7 @@ SERVICE_TYPE_MAP = {
**{service: PipelineConnection for service in PipelineServiceType.__members__},
**{service: MlModelConnection for service in MlModelServiceType.__members__},
**{service: StorageConnection for service in StorageServiceType.__members__},
**{service: SearchConnection for service in SearchServiceType.__members__},
}
SOURCE_CONFIG_CLASS_MAP = {
@ -113,6 +122,7 @@ SOURCE_CONFIG_CLASS_MAP = {
MlModelMetadataConfigType.MlModelMetadata.value: MlModelServiceMetadataPipeline,
DatabaseMetadataConfigType.DatabaseMetadata.value: DatabaseServiceMetadataPipeline,
StorageMetadataConfigType.StorageMetadata.value: StorageServiceMetadataPipeline,
SearchMetadataConfigType.SearchMetadata.value: SearchServiceMetadataPipeline,
}

15
ingestion/src/metadata/ingestion/ometa/ometa_api.py
View File

@ -49,6 +49,7 @@ from metadata.generated.schema.entity.data.mlmodel import MlModel
from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.query import Query
from metadata.generated.schema.entity.data.report import Report
from metadata.generated.schema.entity.data.searchIndex import SearchIndex
from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.policies.policy import Policy
@ -67,6 +68,7 @@ from metadata.generated.schema.entity.services.messagingService import Messaging
from metadata.generated.schema.entity.services.metadataService import MetadataService
from metadata.generated.schema.entity.services.mlmodelService import MlModelService
from metadata.generated.schema.entity.services.pipelineService import PipelineService
from metadata.generated.schema.entity.services.searchService import SearchService
from metadata.generated.schema.entity.services.storageService import StorageService
from metadata.generated.schema.entity.teams.role import Role
from metadata.generated.schema.entity.teams.team import Team
@ -358,6 +360,12 @@ class OpenMetadata(
):
return "/containers"
if issubclass(
entity,
get_args(Union[SearchIndex, self.get_create_entity_type(SearchIndex)]),
):
return "/searchIndexes"
if issubclass(
entity, get_args(Union[Workflow, self.get_create_entity_type(Workflow)])
):
@ -422,11 +430,9 @@ class OpenMetadata(
if issubclass(
entity,
get_args(
Union[StorageService, self.get_create_entity_type(StorageService)]
),
get_args(Union[SearchService, self.get_create_entity_type(SearchService)]),
):
return "/services/storageServices"
return "/services/searchServices"
if issubclass(
entity,
@ -533,6 +539,7 @@ class OpenMetadata(
.replace("testsuite", "testSuite")
.replace("testdefinition", "testDefinition")
.replace("testcase", "testCase")
.replace("searchindex", "searchIndex")
)
class_path = ".".join(

3
ingestion/src/metadata/ingestion/ometa/utils.py
View File

@ -31,6 +31,7 @@ def format_name(name: str) -> str:
return re.sub(r"[" + subs + "]", "_", name)
# pylint: disable=too-many-return-statements
def get_entity_type(
entity: Union[Type[T], str],
) -> str:
@ -54,6 +55,8 @@ def get_entity_type(
return class_name.replace("testsuite", "testSuite")
if "databaseschema" in class_name:
return class_name.replace("databaseschema", "databaseSchema")
if "searchindex" in class_name:
return class_name.replace("searchindex", "searchIndex")
return class_name

57
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -34,6 +34,9 @@ from metadata.generated.schema.api.data.createDatabaseSchema import (
)
from metadata.generated.schema.api.data.createMlModel import CreateMlModelRequest
from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
from metadata.generated.schema.api.data.createSearchIndex import (
CreateSearchIndexRequest,
)
from metadata.generated.schema.api.data.createTable import CreateTableRequest
from metadata.generated.schema.api.data.createTableProfile import (
CreateTableProfileRequest,
@ -77,6 +80,7 @@ from metadata.generated.schema.entity.services.databaseService import DatabaseSe
from metadata.generated.schema.entity.services.messagingService import MessagingService
from metadata.generated.schema.entity.services.mlmodelService import MlModelService
from metadata.generated.schema.entity.services.pipelineService import PipelineService
from metadata.generated.schema.entity.services.searchService import SearchService
from metadata.generated.schema.entity.services.storageService import StorageService
from metadata.generated.schema.entity.teams.team import Team
from metadata.generated.schema.entity.teams.user import User
@ -458,6 +462,34 @@ class SampleDataSource(
)
)
self.storage_service_json = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/storage/service.json",
"r",
encoding=UTF_8,
)
)
self.search_service_json = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/searchIndexes/service.json",
"r",
encoding=UTF_8,
)
)
self.search_service = self.metadata.get_service_or_create(
entity=SearchService,
config=WorkflowSource(**self.search_service_json),
)
self.search_indexes = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/searchIndexes/searchIndexes.json",
"r",
encoding=UTF_8,
)
)
@classmethod
def create(cls, config_dict, metadata_config: OpenMetadataConnection):
"""Create class instance"""
@ -487,6 +519,7 @@ class SampleDataSource(
yield from self.ingest_pipeline_status()
yield from self.ingest_mlmodels()
yield from self.ingest_containers()
yield from self.ingest_search_indexes()
yield from self.ingest_profiles()
yield from self.ingest_test_suite()
yield from self.ingest_test_case()
@ -707,6 +740,30 @@ class SampleDataSource(
sample_data=TopicSampleData(messages=topic["sampleData"]),
)
def ingest_search_indexes(self) -> Iterable[CreateSearchIndexRequest]:
"""
Ingest Sample SearchIndexes
"""
for search_index in self.search_indexes["searchIndexes"]:
search_index["service"] = EntityReference(
id=self.search_service.id, type="searchService"
)
create_search_index = CreateSearchIndexRequest(
name=search_index["name"],
description=search_index["description"],
displayName=search_index["displayName"],
tags=search_index["tags"],
fields=search_index["fields"],
service=self.search_service.fullyQualifiedName,
)
self.status.scanned(
f"SearchIndex Scanned: {create_search_index.name.__root__}"
)
yield create_search_index
# TODO: Add search index sample data
def ingest_looker(self) -> Iterable[Entity]:
"""
Looker sample data

2
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -86,6 +86,7 @@ public final class Entity {
public static final String STORAGE_SERVICE = "storageService";
public static final String MLMODEL_SERVICE = "mlmodelService";
public static final String METADATA_SERVICE = "metadataService";
public static final String SEARCH_SERVICE = "searchService";
//
// Data asset entities
//
@ -99,6 +100,7 @@ public final class Entity {
public static final String CHART = "chart";
public static final String REPORT = "report";
public static final String TOPIC = "topic";
public static final String SEARCH_INDEX = "searchIndex";
public static final String MLMODEL = "mlmodel";
public static final String CONTAINER = "container";
public static final String QUERY = "query";

49
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -79,6 +79,7 @@ import org.openmetadata.schema.entity.data.MlModel;
import org.openmetadata.schema.entity.data.Pipeline;
import org.openmetadata.schema.entity.data.Query;
import org.openmetadata.schema.entity.data.Report;
import org.openmetadata.schema.entity.data.SearchIndex;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.data.Topic;
import org.openmetadata.schema.entity.domains.DataProduct;
@ -91,6 +92,7 @@ import org.openmetadata.schema.entity.services.MessagingService;
import org.openmetadata.schema.entity.services.MetadataService;
import org.openmetadata.schema.entity.services.MlModelService;
import org.openmetadata.schema.entity.services.PipelineService;
import org.openmetadata.schema.entity.services.SearchService;
import org.openmetadata.schema.entity.services.StorageService;
import org.openmetadata.schema.entity.services.connections.TestConnectionDefinition;
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
@ -188,6 +190,9 @@ public interface CollectionDAO {
@CreateSqlObject
MlModelDAO mlModelDAO();
@CreateSqlObject
SearchIndexDAO searchIndexDAO();
@CreateSqlObject
GlossaryDAO glossaryDAO();
@ -233,6 +238,9 @@ public interface CollectionDAO {
@CreateSqlObject
StorageServiceDAO storageServiceDAO();
@CreateSqlObject
SearchServiceDAO searchServiceDAO();
@CreateSqlObject
ContainerDAO containerDAO();
@ -531,6 +539,40 @@ public interface CollectionDAO {
@Define("sqlCondition") String mysqlCond);
}
interface SearchServiceDAO extends EntityDAO<SearchService> {
@Override
default String getTableName() {
return "search_service_entity";
}
@Override
default Class<SearchService> getEntityClass() {
return SearchService.class;
}
@Override
default String getNameHashColumn() {
return "nameHash";
}
}
interface SearchIndexDAO extends EntityDAO<SearchIndex> {
@Override
default String getTableName() {
return "search_index_entity";
}
@Override
default Class<SearchIndex> getEntityClass() {
return SearchIndex.class;
}
@Override
default String getNameHashColumn() {
return "fqnHash";
}
}
interface EntityExtensionDAO {
@ConnectionAwareSqlUpdate(
value =
@ -3397,6 +3439,7 @@ public interface CollectionDAO {
+ "(SELECT COUNT(*) FROM pipeline_entity <cond>) as pipelineCount, "
+ "(SELECT COUNT(*) FROM ml_model_entity <cond>) as mlmodelCount, "
+ "(SELECT COUNT(*) FROM storage_container_entity <cond>) as storageContainerCount, "
+ "(SELECT COUNT(*) FROM search_index_entity <cond>) as searchIndexCount, "
+ "(SELECT COUNT(*) FROM glossary_entity <cond>) as glossaryCount, "
+ "(SELECT COUNT(*) FROM glossary_term_entity <cond>) as glossaryTermCount, "
+ "(SELECT (SELECT COUNT(*) FROM metadata_service_entity <cond>) + "
@ -3405,6 +3448,7 @@ public interface CollectionDAO {
+ "(SELECT COUNT(*) FROM dashboard_service_entity <cond>)+ "
+ "(SELECT COUNT(*) FROM pipeline_service_entity <cond>)+ "
+ "(SELECT COUNT(*) FROM mlmodel_service_entity <cond>)+ "
+ "(SELECT COUNT(*) FROM search_service_entity <cond>)+ "
+ "(SELECT COUNT(*) FROM storage_service_entity <cond>)) as servicesCount, "
+ "(SELECT COUNT(*) FROM user_entity <cond> AND (JSON_EXTRACT(json, '$.isBot') IS NULL OR JSON_EXTRACT(json, '$.isBot') = FALSE)) as userCount, "
+ "(SELECT COUNT(*) FROM team_entity <cond>) as teamCount, "
@ -3418,6 +3462,7 @@ public interface CollectionDAO {
+ "(SELECT COUNT(*) FROM pipeline_entity <cond>) as pipelineCount, "
+ "(SELECT COUNT(*) FROM ml_model_entity <cond>) as mlmodelCount, "
+ "(SELECT COUNT(*) FROM storage_container_entity <cond>) as storageContainerCount, "
+ "(SELECT COUNT(*) FROM search_index_entity <cond>) as searchIndexCount, "
+ "(SELECT COUNT(*) FROM glossary_entity <cond>) as glossaryCount, "
+ "(SELECT COUNT(*) FROM glossary_term_entity <cond>) as glossaryTermCount, "
+ "(SELECT (SELECT COUNT(*) FROM metadata_service_entity <cond>) + "
@ -3426,6 +3471,7 @@ public interface CollectionDAO {
+ "(SELECT COUNT(*) FROM dashboard_service_entity <cond>)+ "
+ "(SELECT COUNT(*) FROM pipeline_service_entity <cond>)+ "
+ "(SELECT COUNT(*) FROM mlmodel_service_entity <cond>)+ "
+ "(SELECT COUNT(*) FROM search_service_entity <cond>)+ "
+ "(SELECT COUNT(*) FROM storage_service_entity <cond>)) as servicesCount, "
+ "(SELECT COUNT(*) FROM user_entity <cond> AND (json#>'{isBot}' IS NULL OR ((json#>'{isBot}')::boolean) = FALSE)) as userCount, "
+ "(SELECT COUNT(*) FROM team_entity <cond>) as teamCount, "
@ -3440,7 +3486,8 @@ public interface CollectionDAO {
+ "(SELECT COUNT(*) FROM dashboard_service_entity <cond>) as dashboardServiceCount, "
+ "(SELECT COUNT(*) FROM pipeline_service_entity <cond>) as pipelineServiceCount, "
+ "(SELECT COUNT(*) FROM mlmodel_service_entity <cond>) as mlModelServiceCount, "
+ "(SELECT COUNT(*) FROM storage_service_entity <cond>) as storageServiceCount")
+ "(SELECT COUNT(*) FROM storage_service_entity <cond>) as storageServiceCount, "
+ "(SELECT COUNT(*) FROM search_service_entity <cond>) as searchServiceCount")
@RegisterRowMapper(ServicesCountRowMapper.class)
ServicesCount getAggregatedServicesCount(@Define("cond") String cond) throws StatementException;

459
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java Normal file
View File

@ -0,0 +1,459 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.schema.type.Include.ALL;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_DOMAIN;
import static org.openmetadata.service.Entity.FIELD_FOLLOWERS;
import static org.openmetadata.service.Entity.FIELD_TAGS;
import static org.openmetadata.service.Entity.SEARCH_SERVICE;
import static org.openmetadata.service.util.EntityUtil.getSearchIndexField;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.function.BiPredicate;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.json.JsonPatch;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.entity.data.SearchIndex;
import org.openmetadata.schema.entity.services.SearchService;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.schema.type.SearchIndexField;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.TaskDetails;
import org.openmetadata.schema.type.searchindex.SearchIndexSampleData;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.resources.feeds.MessageParser;
import org.openmetadata.service.resources.searchindex.SearchIndexResource;
import org.openmetadata.service.security.mask.PIIMasker;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;
public class SearchIndexRepository extends EntityRepository<SearchIndex> {
@Override
public void setFullyQualifiedName(SearchIndex searchIndex) {
searchIndex.setFullyQualifiedName(
FullyQualifiedName.add(searchIndex.getService().getFullyQualifiedName(), searchIndex.getName()));
if (searchIndex.getFields() != null) {
setFieldFQN(searchIndex.getFullyQualifiedName(), searchIndex.getFields());
}
}
public SearchIndexRepository(CollectionDAO dao) {
super(
SearchIndexResource.COLLECTION_PATH, Entity.SEARCH_INDEX, SearchIndex.class, dao.searchIndexDAO(), dao, "", "");
}
@Override
public void prepare(SearchIndex searchIndex) {
SearchService searchService = Entity.getEntity(searchIndex.getService(), "", ALL);
searchIndex.setService(searchService.getEntityReference());
searchIndex.setServiceType(searchService.getServiceType());
// Validate field tags
if (searchIndex.getFields() != null) {
addDerivedFieldTags(searchIndex.getFields());
validateSchemaFieldTags(searchIndex.getFields());
}
}
@Override
public void storeEntity(SearchIndex searchIndex, boolean update) {
// Relationships and fields such as service are derived and not stored as part of json
EntityReference service = searchIndex.getService();
searchIndex.withService(null);
// Don't store fields tags as JSON but build it on the fly based on relationships
List<SearchIndexField> fieldsWithTags = null;
if (searchIndex.getFields() != null) {
fieldsWithTags = searchIndex.getFields();
searchIndex.setFields(cloneWithoutTags(fieldsWithTags));
searchIndex.getFields().forEach(field -> field.setTags(null));
}
store(searchIndex, update);
// Restore the relationships
if (fieldsWithTags != null) {
searchIndex.setFields(fieldsWithTags);
}
searchIndex.withService(service);
}
@Override
public void storeRelationships(SearchIndex searchIndex) {
setService(searchIndex, searchIndex.getService());
}
@Override
public SearchIndex setInheritedFields(SearchIndex searchIndex, Fields fields) {
// If searchIndex does not have domain, then inherit it from parent messaging service
if (fields.contains(FIELD_DOMAIN) && searchIndex.getDomain() == null) {
SearchService service = Entity.getEntity(SEARCH_SERVICE, searchIndex.getService().getId(), "domain", ALL);
searchIndex.withDomain(service.getDomain());
}
return searchIndex;
}
@Override
public SearchIndex setFields(SearchIndex searchIndex, Fields fields) {
searchIndex.setService(getContainer(searchIndex.getId()));
searchIndex.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(searchIndex) : null);
if (searchIndex.getFields() != null) {
getFieldTags(fields.contains(FIELD_TAGS), searchIndex.getFields());
}
return searchIndex;
}
@Override
public SearchIndex clearFields(SearchIndex searchIndex, Fields fields) {
return searchIndex;
}
@Override
public SearchIndexUpdater getUpdater(SearchIndex original, SearchIndex updated, Operation operation) {
return new SearchIndexUpdater(original, updated, operation);
}
public void setService(SearchIndex searchIndex, EntityReference service) {
if (service != null && searchIndex != null) {
addRelationship(
service.getId(), searchIndex.getId(), service.getType(), Entity.SEARCH_INDEX, Relationship.CONTAINS);
searchIndex.setService(service);
}
}
public SearchIndex getSampleData(UUID searchIndexId, boolean authorizePII) {
// Validate the request content
SearchIndex searchIndex = dao.findEntityById(searchIndexId);
SearchIndexSampleData sampleData =
JsonUtils.readValue(
daoCollection.entityExtensionDAO().getExtension(searchIndex.getId().toString(), "searchIndex.sampleData"),
SearchIndexSampleData.class);
searchIndex.setSampleData(sampleData);
setFieldsInternal(searchIndex, Fields.EMPTY_FIELDS);
// Set the fields tags. Will be used to mask the sample data
if (!authorizePII) {
getFieldTags(true, searchIndex.getFields());
searchIndex.setTags(getTags(searchIndex.getFullyQualifiedName()));
return PIIMasker.getSampleData(searchIndex);
}
return searchIndex;
}
@Transaction
public SearchIndex addSampleData(UUID searchIndexId, SearchIndexSampleData sampleData) {
// Validate the request content
SearchIndex searchIndex = daoCollection.searchIndexDAO().findEntityById(searchIndexId);
daoCollection
.entityExtensionDAO()
.insert(
searchIndexId.toString(),
"searchIndex.sampleData",
"searchIndexSampleData",
JsonUtils.pojoToJson(sampleData));
setFieldsInternal(searchIndex, Fields.EMPTY_FIELDS);
return searchIndex.withSampleData(sampleData);
}
private void setFieldFQN(String parentFQN, List<SearchIndexField> fields) {
fields.forEach(
c -> {
String fieldFqn = FullyQualifiedName.add(parentFQN, c.getName());
c.setFullyQualifiedName(fieldFqn);
if (c.getChildren() != null) {
setFieldFQN(fieldFqn, c.getChildren());
}
});
}
private void getFieldTags(boolean setTags, List<SearchIndexField> fields) {
for (SearchIndexField f : listOrEmpty(fields)) {
f.setTags(setTags ? getTags(f.getFullyQualifiedName()) : null);
getFieldTags(setTags, f.getChildren());
}
}
private void addDerivedFieldTags(List<SearchIndexField> fields) {
if (nullOrEmpty(fields)) {
return;
}
for (SearchIndexField field : fields) {
field.setTags(addDerivedTags(field.getTags()));
if (field.getChildren() != null) {
addDerivedFieldTags(field.getChildren());
}
}
}
List<SearchIndexField> cloneWithoutTags(List<SearchIndexField> fields) {
if (nullOrEmpty(fields)) {
return fields;
}
List<SearchIndexField> copy = new ArrayList<>();
fields.forEach(f -> copy.add(cloneWithoutTags(f)));
return copy;
}
private SearchIndexField cloneWithoutTags(SearchIndexField field) {
List<SearchIndexField> children = cloneWithoutTags(field.getChildren());
return new SearchIndexField()
.withDescription(field.getDescription())
.withName(field.getName())
.withDisplayName(field.getDisplayName())
.withFullyQualifiedName(field.getFullyQualifiedName())
.withDataType(field.getDataType())
.withDataTypeDisplay(field.getDataTypeDisplay())
.withChildren(children);
}
private void validateSchemaFieldTags(List<SearchIndexField> fields) {
// Add field level tags by adding tag to field relationship
for (SearchIndexField field : fields) {
checkMutuallyExclusive(field.getTags());
if (field.getChildren() != null) {
validateSchemaFieldTags(field.getChildren());
}
}
}
private void applyTags(List<SearchIndexField> fields) {
// Add field level tags by adding tag to field relationship
for (SearchIndexField field : fields) {
applyTags(field.getTags(), field.getFullyQualifiedName());
if (field.getChildren() != null) {
applyTags(field.getChildren());
}
}
}
@Override
public void applyTags(SearchIndex searchIndex) {
// Add table level tags by adding tag to table relationship
super.applyTags(searchIndex);
if (searchIndex.getFields() != null) {
applyTags(searchIndex.getFields());
}
}
@Override
public List<TagLabel> getAllTags(EntityInterface entity) {
List<TagLabel> allTags = new ArrayList<>();
SearchIndex searchIndex = (SearchIndex) entity;
EntityUtil.mergeTags(allTags, searchIndex.getTags());
List<SearchIndexField> schemaFields = searchIndex.getFields() != null ? searchIndex.getFields() : null;
for (SearchIndexField schemaField : listOrEmpty(schemaFields)) {
EntityUtil.mergeTags(allTags, schemaField.getTags());
}
return allTags;
}
@Override
public void update(TaskDetails task, MessageParser.EntityLink entityLink, String newValue, String user) {
if (entityLink.getFieldName().equals("fields")) {
String schemaName = entityLink.getArrayFieldName();
String childrenSchemaName = "";
if (entityLink.getArrayFieldName().contains(".")) {
String fieldNameWithoutQuotes =
entityLink.getArrayFieldName().substring(1, entityLink.getArrayFieldName().length() - 1);
schemaName = fieldNameWithoutQuotes.substring(0, fieldNameWithoutQuotes.indexOf("."));
childrenSchemaName = fieldNameWithoutQuotes.substring(fieldNameWithoutQuotes.lastIndexOf(".") + 1);
}
SearchIndex searchIndex = getByName(null, entityLink.getEntityFQN(), getFields("tags"), ALL, false);
SearchIndexField schemaField = null;
for (SearchIndexField field : searchIndex.getFields()) {
if (field.getName().equals(schemaName)) {
schemaField = field;
break;
}
}
if (!"".equals(childrenSchemaName) && schemaField != null) {
schemaField = getChildrenSchemaField(schemaField.getChildren(), childrenSchemaName);
}
if (schemaField == null) {
throw new IllegalArgumentException(
CatalogExceptionMessage.invalidFieldName("schema", entityLink.getArrayFieldName()));
}
String origJson = JsonUtils.pojoToJson(searchIndex);
if (EntityUtil.isDescriptionTask(task.getType())) {
schemaField.setDescription(newValue);
} else if (EntityUtil.isTagTask(task.getType())) {
List<TagLabel> tags = JsonUtils.readObjects(newValue, TagLabel.class);
schemaField.setTags(tags);
}
String updatedEntityJson = JsonUtils.pojoToJson(searchIndex);
JsonPatch patch = JsonUtils.getJsonPatch(origJson, updatedEntityJson);
patch(null, searchIndex.getId(), user, patch);
return;
}
super.update(task, entityLink, newValue, user);
}
private static SearchIndexField getChildrenSchemaField(List<SearchIndexField> fields, String childrenSchemaName) {
SearchIndexField childrenSchemaField = null;
for (SearchIndexField field : fields) {
if (field.getName().equals(childrenSchemaName)) {
childrenSchemaField = field;
break;
}
}
if (childrenSchemaField == null) {
for (SearchIndexField field : fields) {
if (field.getChildren() != null) {
childrenSchemaField = getChildrenSchemaField(field.getChildren(), childrenSchemaName);
if (childrenSchemaField != null) {
break;
}
}
}
}
return childrenSchemaField;
}
public static Set<TagLabel> getAllFieldTags(SearchIndexField field) {
Set<TagLabel> tags = new HashSet<>();
if (!listOrEmpty(field.getTags()).isEmpty()) {
tags.addAll(field.getTags());
}
for (SearchIndexField c : listOrEmpty(field.getChildren())) {
tags.addAll(getAllFieldTags(c));
}
return tags;
}
public class SearchIndexUpdater extends EntityUpdater {
public static final String FIELD_DATA_TYPE_DISPLAY = "dataTypeDisplay";
public SearchIndexUpdater(SearchIndex original, SearchIndex updated, Operation operation) {
super(original, updated, operation);
}
@Override
public void entitySpecificUpdate() {
if (updated.getFields() != null) {
updateSearchIndexFields(
"fields",
original.getFields() == null ? null : original.getFields(),
updated.getFields(),
EntityUtil.searchIndexFieldMatch);
}
recordChange("searchIndexSettings", original.getSearchIndexSettings(), updated.getSearchIndexSettings());
}
private void updateSearchIndexFields(
String fieldName,
List<SearchIndexField> origFields,
List<SearchIndexField> updatedFields,
BiPredicate<SearchIndexField, SearchIndexField> fieldMatch) {
List<SearchIndexField> deletedFields = new ArrayList<>();
List<SearchIndexField> addedFields = new ArrayList<>();
recordListChange(fieldName, origFields, updatedFields, addedFields, deletedFields, fieldMatch);
// carry forward tags and description if deletedFields matches added field
Map<String, SearchIndexField> addedFieldMap =
addedFields.stream().collect(Collectors.toMap(SearchIndexField::getName, Function.identity()));
for (SearchIndexField deleted : deletedFields) {
if (addedFieldMap.containsKey(deleted.getName())) {
SearchIndexField addedField = addedFieldMap.get(deleted.getName());
if (nullOrEmpty(addedField.getDescription()) && nullOrEmpty(deleted.getDescription())) {
addedField.setDescription(deleted.getDescription());
}
if (nullOrEmpty(addedField.getTags()) && nullOrEmpty(deleted.getTags())) {
addedField.setTags(deleted.getTags());
}
}
}
// Delete tags related to deleted fields
deletedFields.forEach(deleted -> daoCollection.tagUsageDAO().deleteTagsByTarget(deleted.getFullyQualifiedName()));
// Add tags related to newly added fields
for (SearchIndexField added : addedFields) {
applyTags(added.getTags(), added.getFullyQualifiedName());
}
// Carry forward the user generated metadata from existing fields to new fields
for (SearchIndexField updated : updatedFields) {
// Find stored field matching name, data type and ordinal position
SearchIndexField stored = origFields.stream().filter(c -> fieldMatch.test(c, updated)).findAny().orElse(null);
if (stored == null) { // New field added
continue;
}
updateFieldDescription(stored, updated);
updateFieldDataTypeDisplay(stored, updated);
updateFieldDisplayName(stored, updated);
updateTags(
stored.getFullyQualifiedName(),
EntityUtil.getFieldName(fieldName, updated.getName(), FIELD_TAGS),
stored.getTags(),
updated.getTags());
if (updated.getChildren() != null && stored.getChildren() != null) {
String childrenFieldName = EntityUtil.getFieldName(fieldName, updated.getName());
updateSearchIndexFields(childrenFieldName, stored.getChildren(), updated.getChildren(), fieldMatch);
}
}
majorVersionChange = majorVersionChange || !deletedFields.isEmpty();
}
private void updateFieldDescription(SearchIndexField origField, SearchIndexField updatedField) {
if (operation.isPut() && !nullOrEmpty(origField.getDescription()) && updatedByBot()) {
// Revert the non-empty field description if being updated by a bot
updatedField.setDescription(origField.getDescription());
return;
}
String field = getSearchIndexField(original, origField, FIELD_DESCRIPTION);
recordChange(field, origField.getDescription(), updatedField.getDescription());
}
private void updateFieldDisplayName(SearchIndexField origField, SearchIndexField updatedField) {
if (operation.isPut() && !nullOrEmpty(origField.getDescription()) && updatedByBot()) {
// Revert the non-empty field description if being updated by a bot
updatedField.setDisplayName(origField.getDisplayName());
return;
}
String field = getSearchIndexField(original, origField, FIELD_DISPLAY_NAME);
recordChange(field, origField.getDisplayName(), updatedField.getDisplayName());
}
private void updateFieldDataTypeDisplay(SearchIndexField origField, SearchIndexField updatedField) {
if (operation.isPut() && !nullOrEmpty(origField.getDataTypeDisplay()) && updatedByBot()) {
// Revert the non-empty field dataTypeDisplay if being updated by a bot
updatedField.setDataTypeDisplay(origField.getDataTypeDisplay());
return;
}
String field = getSearchIndexField(original, origField, FIELD_DATA_TYPE_DISPLAY);
recordChange(field, origField.getDataTypeDisplay(), updatedField.getDataTypeDisplay());
}
}
}

19
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchServiceRepository.java Normal file
View File

@ -0,0 +1,19 @@
package org.openmetadata.service.jdbi3;
import org.openmetadata.schema.entity.services.SearchService;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.type.SearchConnection;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.services.storage.StorageServiceResource;
public class SearchServiceRepository extends ServiceEntityRepository<SearchService, SearchConnection> {
public SearchServiceRepository(CollectionDAO dao) {
super(
StorageServiceResource.COLLECTION_PATH,
Entity.SEARCH_SERVICE,
dao,
dao.searchServiceDAO(),
SearchConnection.class,
ServiceType.SEARCH);
}
}

480
openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java Normal file
View File

@ -0,0 +1,480 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.resources.searchindex;
import static org.openmetadata.common.utils.CommonUtil.listOf;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.util.List;
import java.util.UUID;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import org.openmetadata.schema.api.data.CreateSearchIndex;
import org.openmetadata.schema.api.data.RestoreEntity;
import org.openmetadata.schema.entity.data.SearchIndex;
import org.openmetadata.schema.type.ChangeEvent;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.searchindex.SearchIndexSampleData;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.SearchIndexRepository;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.security.policyevaluator.ResourceContext;
import org.openmetadata.service.util.ResultList;
@Path("/v1/searchIndexes")
@Tag(
name = "SearchIndex",
description = "A `SearchIndex` is a index mapping for indexing documents in a `Search Service`.")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "searchIndexes")
public class SearchIndexResource extends EntityResource<SearchIndex, SearchIndexRepository> {
public static final String COLLECTION_PATH = "v1/searchIndexes/";
static final String FIELDS = "owner,followers,tags,extension,domain,dataProducts";
@Override
public SearchIndex addHref(UriInfo uriInfo, SearchIndex searchIndex) {
Entity.withHref(uriInfo, searchIndex.getOwner());
Entity.withHref(uriInfo, searchIndex.getService());
Entity.withHref(uriInfo, searchIndex.getFollowers());
Entity.withHref(uriInfo, searchIndex.getDomain());
return searchIndex;
}
public SearchIndexResource(CollectionDAO dao, Authorizer authorizer) {
super(SearchIndex.class, new SearchIndexRepository(dao), authorizer);
}
@Override
protected List<MetadataOperation> getEntitySpecificOperations() {
addViewOperation("sampleData", MetadataOperation.VIEW_SAMPLE_DATA);
return listOf(MetadataOperation.VIEW_SAMPLE_DATA, MetadataOperation.EDIT_SAMPLE_DATA);
}
public static class SearchIndexList extends ResultList<SearchIndex> {
/* Required for serde */
}
@GET
@Operation(
operationId = "listSearchIndexes",
summary = "List searchIndexes",
description =
"Get a list of SearchIndexes, optionally filtered by `service` it belongs to. Use `fields` "
+ "parameter to get only necessary fields. Use cursor-based pagination to limit the number "
+ "entries in the list using `limit` and `before` or `after` query params.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of SearchIndexes",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndexList.class)))
})
public ResultList<SearchIndex> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Filter SearchIndexes by service name",
schema = @Schema(type = "string", example = "ElasticSearchWestCoast"))
@QueryParam("service")
String serviceParam,
@Parameter(description = "Limit the number SearchIndexes returned. (1 to 1000000, default = " + "10)")
@DefaultValue("10")
@QueryParam("limit")
@Min(0)
@Max(1000000)
int limitParam,
@Parameter(description = "Returns list of SearchIndexes before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(description = "Returns list of SearchIndexes after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
ListFilter filter = new ListFilter(include).addQueryParam("service", serviceParam);
return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after);
}
@GET
@Path("/{id}/versions")
@Operation(
operationId = "listAllSearchIndexVersion",
summary = "List SearchIndex versions",
description = "Get a list of all the versions of a SearchIndex identified by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of SearchIndex versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) {
return super.listVersionsInternal(securityContext, id);
}
@GET
@Path("/{id}")
@Operation(
summary = "Get a SearchIndex by id",
description = "Get a SearchIndex by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The SearchIndex",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))),
@ApiResponse(responseCode = "404", description = "SearchIndex for instance {id} is not found")
})
public SearchIndex get(
@Context UriInfo uriInfo,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
return getInternal(uriInfo, securityContext, id, fieldsParam, include);
}
@GET
@Path("/name/{fqn}")
@Operation(
operationId = "getSearchIndexByFQN",
summary = "Get a SearchIndex by fully qualified name",
description = "Get a SearchIndex by fully qualified name.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The SearchIndex",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))),
@ApiResponse(responseCode = "404", description = "SearchIndex for instance {fqn} is not found")
})
public SearchIndex getByName(
@Context UriInfo uriInfo,
@Parameter(description = "Fully qualified name of the SearchIndex", schema = @Schema(type = "string"))
@PathParam("fqn")
String fqn,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
operationId = "getSpecificSearchIndexVersion",
summary = "Get a version of the SearchIndex",
description = "Get a version of the SearchIndex by given `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "SearchIndex",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))),
@ApiResponse(
responseCode = "404",
description = "SearchIndex for instance {id} and version {version} is " + "not found")
})
public SearchIndex getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@Parameter(
description = "SearchIndex version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version) {
return super.getVersionInternal(securityContext, id, version);
}
@POST
@Operation(
operationId = "createSearchIndex",
summary = "Create a SearchIndex",
description = "Create a SearchIndex under an existing `service`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The SearchIndex",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchIndex create) {
SearchIndex searchIndex = getSearchIndex(create, securityContext.getUserPrincipal().getName());
return create(uriInfo, securityContext, searchIndex);
}
@PATCH
@Path("/{id}")
@Operation(
operationId = "patchSearchIndex",
summary = "Update a SearchIndex",
description = "Update an existing SearchIndex using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response updateDescription(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch) {
return patchInternal(uriInfo, securityContext, id, patch);
}
@PUT
@Operation(
operationId = "createOrUpdateSearchIndex",
summary = "Update SearchIndex",
description = "Create a SearchIndex, it it does not exist or update an existing SearchIndex.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The updated SearchIndex ",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class)))
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchIndex create) {
SearchIndex searchIndex = getSearchIndex(create, securityContext.getUserPrincipal().getName());
return createOrUpdate(uriInfo, securityContext, searchIndex);
}
@PUT
@Path("/{id}/sampleData")
@Operation(
operationId = "addSampleData",
summary = "Add sample data",
description = "Add sample data to the searchIndex.",
responses = {
@ApiResponse(
responseCode = "200",
description = "The SearchIndex",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))),
})
public SearchIndex addSampleData(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@Valid SearchIndexSampleData sampleData) {
OperationContext operationContext = new OperationContext(entityType, MetadataOperation.EDIT_SAMPLE_DATA);
authorizer.authorize(securityContext, operationContext, getResourceContextById(id));
SearchIndex searchIndex = repository.addSampleData(id, sampleData);
return addHref(uriInfo, searchIndex);
}
@GET
@Path("/{id}/sampleData")
@Operation(
operationId = "getSampleData",
summary = "Get sample data",
description = "Get sample data from the SearchIndex.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Successfully obtained the SampleData for SearchIndex",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class)))
})
public SearchIndex getSampleData(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) {
OperationContext operationContext = new OperationContext(entityType, MetadataOperation.VIEW_SAMPLE_DATA);
ResourceContext resourceContext = getResourceContextById(id);
authorizer.authorize(securityContext, operationContext, resourceContext);
boolean authorizePII = authorizer.authorizePII(securityContext, resourceContext.getOwner());
SearchIndex searchIndex = repository.getSampleData(id, authorizePII);
return addHref(uriInfo, searchIndex);
}
@PUT
@Path("/{id}/followers")
@Operation(
operationId = "addFollower",
summary = "Add a follower",
description = "Add a user identified by `userId` as followed of this SearchIndex",
responses = {
@ApiResponse(
responseCode = "200",
description = "OK",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = ChangeEvent.class))),
@ApiResponse(responseCode = "404", description = "SearchIndex for instance {id} is not found")
})
public Response addFollower(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@Parameter(description = "Id of the user to be added as follower", schema = @Schema(type = "UUID")) UUID userId) {
return repository.addFollower(securityContext.getUserPrincipal().getName(), id, userId).toResponse();
}
@DELETE
@Path("/{id}/followers/{userId}")
@Operation(
summary = "Remove a follower",
description = "Remove the user identified `userId` as a follower of the SearchIndex.",
responses = {
@ApiResponse(
responseCode = "200",
description = "OK",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = ChangeEvent.class)))
})
public Response deleteFollower(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@Parameter(description = "Id of the user being removed as follower", schema = @Schema(type = "string"))
@PathParam("userId")
String userId) {
return repository
.deleteFollower(securityContext.getUserPrincipal().getName(), id, UUID.fromString(userId))
.toResponse();
}
@DELETE
@Path("/{id}")
@Operation(
operationId = "deleteSearchIndex",
summary = "Delete a SearchIndex by id",
description = "Delete a SearchIndex by `id`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "SearchIndex for instance {id} is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Hard delete the entity. (Default = `false`)")
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) {
return delete(uriInfo, securityContext, id, false, hardDelete);
}
@DELETE
@Path("/name/{fqn}")
@Operation(
operationId = "deleteSearchIndexByFQN",
summary = "Delete a SearchIndex by fully qualified name",
description = "Delete a SearchIndex by `fullyQualifiedName`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "SearchIndex for instance {fqn} is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Hard delete the entity. (Default = `false`)")
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Fully qualified name of the SearchIndex", schema = @Schema(type = "string"))
@PathParam("fqn")
String fqn) {
return deleteByName(uriInfo, securityContext, fqn, false, hardDelete);
}
@PUT
@Path("/restore")
@Operation(
operationId = "restore",
summary = "Restore a soft deleted SearchIndex",
description = "Restore a soft deleted SearchIndex.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Successfully restored the SearchIndex. ",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class)))
})
public Response restoreSearchIndex(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid RestoreEntity restore) {
return restoreEntity(uriInfo, securityContext, restore.getId());
}
private SearchIndex getSearchIndex(CreateSearchIndex create, String user) {
return copy(new SearchIndex(), create, user)
.withService(getEntityReference(Entity.SEARCH_SERVICE, create.getService()))
.withFields(create.getFields())
.withSearchIndexSettings(create.getSearchIndexSettings())
.withTags(create.getTags());
}
}

436
openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java Normal file
View File

@ -0,0 +1,436 @@
package org.openmetadata.service.resources.services.searchIndexes;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.PATCH;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.api.data.RestoreEntity;
import org.openmetadata.schema.api.services.CreateSearchService;
import org.openmetadata.schema.entity.services.SearchService;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.entity.services.connections.TestConnectionResult;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.SearchConnection;
import org.openmetadata.schema.utils.EntityInterfaceUtil;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.SearchServiceRepository;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.services.ServiceEntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@Slf4j
@Path("/v1/services/searchServices")
@Tag(
name = "Search Services",
description = "APIs related `Search Service` entities, such as ElasticSearch, OpenSearch.")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "searchServices")
public class SearchServiceResource
extends ServiceEntityResource<SearchService, SearchServiceRepository, SearchConnection> {
public static final String COLLECTION_PATH = "v1/services/searchServices/";
static final String FIELDS = "pipelines,owner,tags,domain";
@Override
public SearchService addHref(UriInfo uriInfo, SearchService service) {
service.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, service.getId()));
Entity.withHref(uriInfo, service.getOwner());
Entity.withHref(uriInfo, service.getPipelines());
return service;
}
public SearchServiceResource(CollectionDAO dao, Authorizer authorizer) {
super(SearchService.class, new SearchServiceRepository(dao), authorizer, ServiceType.SEARCH);
}
@Override
protected List<MetadataOperation> getEntitySpecificOperations() {
addViewOperation("pipelines", MetadataOperation.VIEW_BASIC);
return null;
}
public static class SearchServiceList extends ResultList<SearchService> {
/* Required for serde */
}
@GET
@Operation(
operationId = "listSearchServices",
summary = "List search services",
description = "Get a list of search services.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of search service instances",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = SearchServiceResource.SearchServiceList.class)))
})
public ResultList<SearchService> list(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam,
@Parameter(description = "Returns list of search services before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(description = "Returns list of search services after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
RestUtil.validateCursors(before, after);
EntityUtil.Fields fields = getFields(fieldsParam);
ResultList<SearchService> searchServices;
ListFilter filter = new ListFilter(include);
if (before != null) {
searchServices = repository.listBefore(uriInfo, fields, filter, limitParam, before);
} else {
searchServices = repository.listAfter(uriInfo, fields, filter, limitParam, after);
}
return addHref(uriInfo, decryptOrNullify(securityContext, searchServices));
}
@GET
@Path("/{id}")
@Operation(
operationId = "getSearchServiceByID",
summary = "Get an search service",
description = "Get an search service by `id`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "search service instance",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))),
@ApiResponse(responseCode = "404", description = "search service for instance {id} is not found")
})
public SearchService get(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") UUID id,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
SearchService searchService = getInternal(uriInfo, securityContext, id, fieldsParam, include);
return decryptOrNullify(securityContext, searchService);
}
@GET
@Path("/name/{name}")
@Operation(
operationId = "getSearchServiceByFQN",
summary = "Get search service by name",
description = "Get a search service by the service `name`.",
responses = {
@ApiResponse(
responseCode = "200",
description = "search service instance",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))),
@ApiResponse(responseCode = "404", description = "search service for instance {id} is not found")
})
public SearchService getByName(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("name") String name,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@Parameter(
description = "Include all, deleted, or non-deleted entities.",
schema = @Schema(implementation = Include.class))
@QueryParam("include")
@DefaultValue("non-deleted")
Include include) {
SearchService searchService =
getByNameInternal(uriInfo, securityContext, EntityInterfaceUtil.quoteName(name), fieldsParam, include);
return decryptOrNullify(securityContext, searchService);
}
@PUT
@Path("/{id}/testConnectionResult")
@Operation(
operationId = "addTestConnectionResult",
summary = "Add test connection result",
description = "Add test connection result to the service.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Successfully updated the service",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class)))
})
public SearchService addTestConnectionResult(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Id of the service", schema = @Schema(type = "UUID")) @PathParam("id") UUID id,
@Valid TestConnectionResult testConnectionResult) {
OperationContext operationContext = new OperationContext(entityType, MetadataOperation.CREATE);
authorizer.authorize(securityContext, operationContext, getResourceContextById(id));
SearchService service = repository.addTestConnectionResult(id, testConnectionResult);
return decryptOrNullify(securityContext, service);
}
@GET
@Path("/{id}/versions")
@Operation(
operationId = "listAllSearchServiceVersion",
summary = "List search service versions",
description = "Get a list of all the versions of an search service identified by `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of search service versions",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "search service Id", schema = @Schema(type = "string")) @PathParam("id") UUID id) {
EntityHistory entityHistory = super.listVersionsInternal(securityContext, id);
List<Object> versions =
entityHistory.getVersions().stream()
.map(
json -> {
try {
SearchService searchService = JsonUtils.readValue((String) json, SearchService.class);
return JsonUtils.pojoToJson(decryptOrNullify(securityContext, searchService));
} catch (Exception e) {
return json;
}
})
.collect(Collectors.toList());
entityHistory.setVersions(versions);
return entityHistory;
}
@GET
@Path("/{id}/versions/{version}")
@Operation(
operationId = "getSpecificSearchServiceVersion",
summary = "Get a version of the search service",
description = "Get a version of the search service by given `id`",
responses = {
@ApiResponse(
responseCode = "200",
description = "search service",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))),
@ApiResponse(
responseCode = "404",
description = "Object store service for instance {id} and version " + "{version} is not found")
})
public SearchService getVersion(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "search service Id", schema = @Schema(type = "string")) @PathParam("id") UUID id,
@Parameter(
description = "search service version number in the form `major`" + ".`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version")
String version) {
SearchService searchService = super.getVersionInternal(securityContext, id, version);
return decryptOrNullify(securityContext, searchService);
}
@POST
@Operation(
operationId = "createSearchService",
summary = "Create search service",
description = "Create a new search service.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Search service instance",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response create(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchService create) {
SearchService service = getService(create, securityContext.getUserPrincipal().getName());
Response response = create(uriInfo, securityContext, service);
decryptOrNullify(securityContext, (SearchService) response.getEntity());
return response;
}
@PUT
@Operation(
operationId = "createOrUpdateSearchService",
summary = "Update search service",
description = "Update an existing or create a new search service.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Object store service instance",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
public Response createOrUpdate(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchService update) {
SearchService service = getService(update, securityContext.getUserPrincipal().getName());
Response response = createOrUpdate(uriInfo, securityContext, unmask(service));
decryptOrNullify(securityContext, (SearchService) response.getEntity());
return response;
}
@PATCH
@Path("/{id}")
@Operation(
operationId = "patchSearchService",
summary = "Update an search service",
description = "Update an existing search service using JsonPatch.",
externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902"))
@Consumes(MediaType.APPLICATION_JSON_PATCH_JSON)
public Response patch(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@PathParam("id") UUID id,
@RequestBody(
description = "JsonPatch with array of operations",
content =
@Content(
mediaType = MediaType.APPLICATION_JSON_PATCH_JSON,
examples = {
@ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]")
}))
JsonPatch patch) {
return patchInternal(uriInfo, securityContext, id, patch);
}
@DELETE
@Path("/{id}")
@Operation(
operationId = "deleteSearchService",
summary = "Delete an search service",
description = "Delete an search services. If containers belong the service, it can't be " + "deleted.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "SearchService service for instance {id} " + "is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Recursively delete this entity and it's children. (Default `false`)")
@DefaultValue("false")
@QueryParam("recursive")
boolean recursive,
@Parameter(description = "Hard delete the entity. (Default = `false`)")
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Id of the search service", schema = @Schema(type = "string")) @PathParam("id")
UUID id) {
return delete(uriInfo, securityContext, id, recursive, hardDelete);
}
@DELETE
@Path("/name/{fqn}")
@Operation(
operationId = "deleteSearchServiceByFQN",
summary = "Delete an SearchService by fully qualified name",
description = "Delete an SearchService by `fullyQualifiedName`.",
responses = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "SearchService for instance {fqn} is not found")
})
public Response delete(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "Hard delete the entity. (Default = `false`)")
@QueryParam("hardDelete")
@DefaultValue("false")
boolean hardDelete,
@Parameter(description = "Name of the SearchService", schema = @Schema(type = "string")) @PathParam("fqn")
String fqn) {
return deleteByName(uriInfo, securityContext, EntityInterfaceUtil.quoteName(fqn), false, hardDelete);
}
@PUT
@Path("/restore")
@Operation(
operationId = "restore",
summary = "Restore a soft deleted SearchService.",
description = "Restore a soft deleted SearchService.",
responses = {
@ApiResponse(
responseCode = "200",
description = "Successfully restored the SearchService.",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class)))
})
public Response restoreSearchService(
@Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid RestoreEntity restore) {
return restoreEntity(uriInfo, securityContext, restore.getId());
}
private SearchService getService(CreateSearchService create, String user) {
return copy(new SearchService(), create, user)
.withServiceType(create.getServiceType())
.withConnection(create.getConnection());
}
@Override
protected SearchService nullifyConnection(SearchService service) {
return service.withConnection(null);
}
@Override
protected String extractServiceType(SearchService service) {
return service.getServiceType().value();
}
}

22
openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java
View File

@ -13,6 +13,7 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import javax.ws.rs.core.SecurityContext;
import org.openmetadata.schema.entity.data.Query;
import org.openmetadata.schema.entity.data.SearchIndex;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.data.Topic;
import org.openmetadata.schema.tests.TestCase;
@ -21,6 +22,7 @@ import org.openmetadata.schema.type.Field;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.TableData;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.searchindex.SearchIndexSampleData;
import org.openmetadata.schema.type.topic.TopicSampleData;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.ColumnUtil;
@ -99,6 +101,22 @@ public class PIIMasker {
return topic;
}
public static SearchIndex getSampleData(SearchIndex searchIndex) {
SearchIndexSampleData sampleData = searchIndex.getSampleData();
// If we don't have sample data, there's nothing to do
if (sampleData == null) {
return searchIndex;
}
if (hasPiiSensitiveTag(searchIndex)) {
sampleData.setMessages(List.of(MASKED_VALUE));
searchIndex.setSampleData(sampleData);
}
return searchIndex;
}
public static Table getTableProfile(Table table) {
for (Column column : table.getColumns()) {
if (hasPiiSensitiveTag(column)) {
@ -190,6 +208,10 @@ public class PIIMasker {
return table.getTags().stream().map(TagLabel::getTagFQN).anyMatch(SENSITIVE_PII_TAG::equals);
}
private static boolean hasPiiSensitiveTag(SearchIndex searchIndex) {
return searchIndex.getTags().stream().map(TagLabel::getTagFQN).anyMatch(SENSITIVE_PII_TAG::equals);
}
/*
Check if the Topic is flagged as PII or any of its fields
*/

14
openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java
View File

@ -42,6 +42,7 @@ import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.api.data.TermReference;
import org.openmetadata.schema.entity.classification.Tag;
import org.openmetadata.schema.entity.data.GlossaryTerm;
import org.openmetadata.schema.entity.data.SearchIndex;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.data.Topic;
import org.openmetadata.schema.entity.policies.accessControl.Rule;
@ -129,6 +130,10 @@ public final class EntityUtil {
(field1, field2) ->
field1.getName().equalsIgnoreCase(field2.getName()) && field1.getDataType() == field2.getDataType();
public static final BiPredicate<SearchIndexField, SearchIndexField> searchIndexFieldMatch =
(field1, field2) ->
field1.getName().equalsIgnoreCase(field2.getName()) && field1.getDataType() == field2.getDataType();
private EntityUtil() {}
/** Validate that JSON payload can be turned into POJO object */
@ -371,6 +376,15 @@ public final class EntityUtil {
? FullyQualifiedName.build("schemaFields", localFieldName)
: FullyQualifiedName.build("schemaFields", localFieldName, fieldName);
}
/** Return searchIndex field name of format "fields".fieldName.fieldName */
public static String getSearchIndexField(SearchIndex searchIndex, SearchIndexField field, String fieldName) {
// Remove topic FQN from schemaField FQN to get the local name
String localFieldName =
EntityUtil.getLocalColumnName(searchIndex.getFullyQualifiedName(), field.getFullyQualifiedName());
return fieldName == null
? FullyQualifiedName.build("fields", localFieldName)
: FullyQualifiedName.build("fields", localFieldName, fieldName);
}
/** Return rule field name of format "rules".ruleName.ruleFieldName */
public static String getRuleField(Rule rule, String ruleField) {

4
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -197,6 +197,7 @@ import org.openmetadata.service.resources.services.MessagingServiceResourceTest;
import org.openmetadata.service.resources.services.MetadataServiceResourceTest;
import org.openmetadata.service.resources.services.MlModelServiceResourceTest;
import org.openmetadata.service.resources.services.PipelineServiceResourceTest;
import org.openmetadata.service.resources.services.SearchServiceResourceTest;
import org.openmetadata.service.resources.services.StorageServiceResourceTest;
import org.openmetadata.service.resources.tags.TagResourceTest;
import org.openmetadata.service.resources.teams.RoleResourceTest;
@ -292,6 +293,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
public static EntityReference MLFLOW_REFERENCE;
public static EntityReference S3_OBJECT_STORE_SERVICE_REFERENCE;
public static EntityReference ELASTICSEARCH_SEARCH_SERVICE_REFERENCE;
public static EntityReference OPENSEARCH_SEARCH_SERVICE_REFERENCE;
public static EntityReference AMUNDSEN_SERVICE_REFERENCE;
public static EntityReference ATLAS_SERVICE_REFERENCE;
@ -415,6 +418,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
new DashboardServiceResourceTest().setupDashboardServices(test);
new MlModelServiceResourceTest().setupMlModelServices(test);
new StorageServiceResourceTest().setupStorageService(test);
new SearchServiceResourceTest().setupSearchService(test);
new MetadataServiceResourceTest().setupMetadataServices();
new TableResourceTest().setupDatabaseSchemas(test);
new TestSuiteResourceTest().setupTestSuites(test);

426
openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java Normal file
View File

@ -0,0 +1,426 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.resources.searchindex;
import static java.util.Collections.singletonList;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.common.utils.CommonUtil.listOf;
import static org.openmetadata.service.Entity.FIELD_OWNER;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.assertListNotNull;
import static org.openmetadata.service.util.TestUtils.assertListNull;
import static org.openmetadata.service.util.TestUtils.assertResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response.Status;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.schema.api.data.CreateSearchIndex;
import org.openmetadata.schema.api.services.CreateSearchService;
import org.openmetadata.schema.entity.data.SearchIndex;
import org.openmetadata.schema.entity.services.SearchService;
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.SearchIndexDataType;
import org.openmetadata.schema.type.SearchIndexField;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.searchindex.SearchIndexSampleData;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.services.SearchServiceResourceTest;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.util.TestUtils;
import org.openmetadata.service.util.TestUtils.UpdateType;
@Slf4j
public class SearchIndexResourceTest extends EntityResourceTest<SearchIndex, CreateSearchIndex> {
public static final List<SearchIndexField> SEARCH_INDEX_FIELDS =
Arrays.asList(
getField("id", SearchIndexDataType.KEYWORD, null),
getField("name", SearchIndexDataType.KEYWORD, null),
getField("address", SearchIndexDataType.TEXT, null));
public SearchIndexResourceTest() {
super(
Entity.SEARCH_INDEX,
SearchIndex.class,
SearchIndexResource.SearchIndexList.class,
"searchIndexes",
SearchIndexResource.FIELDS);
supportsSearchIndex = true;
}
@Test
void post_searchIndexWithoutRequiredFields_4xx(TestInfo test) {
// Service is required field
assertResponse(
() -> createEntity(createRequest(test).withService(null), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[service must not be null]");
// Partitions is required field
assertResponse(
() -> createEntity(createRequest(test).withFields(null), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[fields must not be null]");
}
@Test
void post_searchIndexWithDifferentService_200_ok(TestInfo test) throws IOException {
String[] differentServices = {
ELASTICSEARCH_SEARCH_SERVICE_REFERENCE.getName(), OPENSEARCH_SEARCH_SERVICE_REFERENCE.getName()
};
// Create searchIndex for each service and test APIs
for (String service : differentServices) {
createAndCheckEntity(createRequest(test).withService(service), ADMIN_AUTH_HEADERS);
// List searchIndexes by filtering on service name and ensure right searchIndexes in the response
Map<String, String> queryParams = new HashMap<>();
queryParams.put("service", service);
ResultList<SearchIndex> list = listEntities(queryParams, ADMIN_AUTH_HEADERS);
for (SearchIndex searchIndex : list.getData()) {
assertEquals(service, searchIndex.getService().getName());
}
}
}
@Test
void put_searchIndexAttributes_200_ok(TestInfo test) throws IOException {
ArrayList<SearchIndexField> fields =
new ArrayList<>(
Arrays.asList(
new SearchIndexField().withName("name").withDataType(SearchIndexDataType.TEXT),
new SearchIndexField().withName("displayName").withDataType(SearchIndexDataType.KEYWORD)));
List<SearchIndexField> searchIndexFields =
Arrays.asList(
new SearchIndexField()
.withName("tableSearchIndex")
.withDataType(SearchIndexDataType.NESTED)
.withChildren(fields));
CreateSearchIndex createSearchIndex = createRequest(test).withOwner(USER1_REF).withFields(searchIndexFields);
SearchIndex searchIndex = createEntity(createSearchIndex, ADMIN_AUTH_HEADERS);
ChangeDescription change = getChangeDescription(searchIndex.getVersion());
// Patch and update the searchIndex
fields.add(new SearchIndexField().withName("updatedBy").withDataType(SearchIndexDataType.KEYWORD));
List<SearchIndexField> updatedSearchIndexFields =
List.of(
new SearchIndexField()
.withName("tableSearchIndex")
.withChildren(fields)
.withDataType(SearchIndexDataType.NESTED));
createSearchIndex.withOwner(TEAM11_REF).withDescription("searchIndex").withFields(updatedSearchIndexFields);
SearchIndexField addedField = fields.get(2);
addedField.setFullyQualifiedName(
searchIndex.getFields().get(0).getFullyQualifiedName() + "." + addedField.getName());
fieldUpdated(change, FIELD_OWNER, USER1_REF, TEAM11_REF);
fieldUpdated(change, "description", "", "searchIndex");
fieldAdded(change, "fields.tableSearchIndex", JsonUtils.pojoToJson(List.of(addedField)));
updateAndCheckEntity(createSearchIndex, Status.OK, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
}
@Test
void put_searchIndexFields_200_ok(TestInfo test) throws IOException {
List<SearchIndexField> fields =
Arrays.asList(
getField("id", SearchIndexDataType.KEYWORD, null),
getField("first_name", SearchIndexDataType.KEYWORD, null),
getField("last_name", SearchIndexDataType.TEXT, null),
getField("email", SearchIndexDataType.KEYWORD, null),
getField("address_line_1", SearchIndexDataType.ARRAY, null),
getField("address_line_2", SearchIndexDataType.TEXT, null),
getField("post_code", SearchIndexDataType.TEXT, null),
getField("county", SearchIndexDataType.TEXT, PERSONAL_DATA_TAG_LABEL));
CreateSearchIndex createSearchIndex = createRequest(test).withOwner(USER1_REF).withFields(fields);
// update the searchIndex
SearchIndex searchIndex = createEntity(createSearchIndex, ADMIN_AUTH_HEADERS);
searchIndex = getEntity(searchIndex.getId(), ADMIN_AUTH_HEADERS);
assertFields(fields, searchIndex.getFields());
}
@Test
void patch_searchIndexAttributes_200_ok(TestInfo test) throws IOException {
List<SearchIndexField> fields =
Arrays.asList(
getField("id", SearchIndexDataType.KEYWORD, null),
getField("first_name", SearchIndexDataType.KEYWORD, null),
getField("last_name", SearchIndexDataType.TEXT, null),
getField("email", SearchIndexDataType.KEYWORD, null),
getField("address_line_1", SearchIndexDataType.ARRAY, null),
getField("address_line_2", SearchIndexDataType.TEXT, null),
getField("post_code", SearchIndexDataType.TEXT, null),
getField("county", SearchIndexDataType.TEXT, PERSONAL_DATA_TAG_LABEL));
CreateSearchIndex createSearchIndex = createRequest(test).withOwner(USER1_REF).withFields(fields);
SearchIndex searchIndex = createEntity(createSearchIndex, ADMIN_AUTH_HEADERS);
String origJson = JsonUtils.pojoToJson(searchIndex);
List<SearchIndexField> updatedFields =
Arrays.asList(
getField("id", SearchIndexDataType.KEYWORD, null),
getField("first_name", SearchIndexDataType.KEYWORD, null),
getField("last_name", SearchIndexDataType.TEXT, null),
getField("email", SearchIndexDataType.KEYWORD, null),
getField("address_line_1", SearchIndexDataType.ARRAY, null),
getField("address_line_2", SearchIndexDataType.TEXT, null),
getField("post_code", SearchIndexDataType.TEXT, null),
getField("county", SearchIndexDataType.TEXT, PERSONAL_DATA_TAG_LABEL),
getField("phone", SearchIndexDataType.TEXT, PERSONAL_DATA_TAG_LABEL));
searchIndex.withOwner(TEAM11_REF).withFields(updatedFields);
SearchIndexField addedField = updatedFields.get(updatedFields.size() - 1);
addedField.setFullyQualifiedName(searchIndex.getFullyQualifiedName() + "." + addedField.getName());
ChangeDescription change = getChangeDescription(searchIndex.getVersion());
fieldUpdated(change, FIELD_OWNER, USER1_REF, TEAM11_REF);
fieldAdded(change, "fields", JsonUtils.pojoToJson(List.of(addedField)));
patchEntityAndCheck(searchIndex, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
}
@Test
void test_mutuallyExclusiveTags(TestInfo testInfo) {
// Apply mutually exclusive tags to a table
List<SearchIndexField> fields =
Arrays.asList(
getField("id", SearchIndexDataType.KEYWORD, null),
getField("first_name", SearchIndexDataType.KEYWORD, null),
getField("last_name", SearchIndexDataType.TEXT, null),
getField("email", SearchIndexDataType.KEYWORD, null));
CreateSearchIndex create =
createRequest(testInfo)
.withTags(List.of(TIER1_TAG_LABEL, TIER2_TAG_LABEL))
.withOwner(USER1_REF)
.withFields(fields);
// Apply mutually exclusive tags to a searchIndex
assertResponse(
() -> createEntity(create, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL));
// Apply mutually exclusive tags to a searchIndex field
CreateSearchIndex create1 = createRequest(testInfo, 1).withOwner(USER1_REF);
SearchIndexField field =
getField("first_name", SearchIndexDataType.TEXT, null).withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL));
create1.withFields(List.of(field));
assertResponse(
() -> createEntity(create1, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL));
// Apply mutually exclusive tags to a searchIndexes's nested field
CreateSearchIndex create2 = createRequest(testInfo, 1).withOwner(USER1_REF);
SearchIndexField nestedField =
getField("testNested", SearchIndexDataType.TEXT, null).withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL));
SearchIndexField field1 = getField("test", SearchIndexDataType.NESTED, null).withChildren(List.of(nestedField));
create2.withFields(List.of(field1));
assertResponse(
() -> createEntity(create2, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL));
}
@Test
void put_searchIndexSampleData_200(TestInfo test) throws IOException {
List<SearchIndexField> fields =
Arrays.asList(
getField("email", SearchIndexDataType.KEYWORD, null),
getField("firstName", SearchIndexDataType.KEYWORD, null),
getField("lastName", SearchIndexDataType.TEXT, null));
SearchIndex searchIndex = createAndCheckEntity(createRequest(test).withFields(fields), ADMIN_AUTH_HEADERS);
List<String> messages =
Arrays.asList(
"{\"email\": \"email1@email.com\", \"firstName\": \"Bob\", \"lastName\": \"Jones\"}",
"{\"email\": \"email2@email.com\", \"firstName\": \"Test\", \"lastName\": \"Jones\"}",
"{\"email\": \"email3@email.com\", \"firstName\": \"Bob\", \"lastName\": \"Jones\"}");
SearchIndexSampleData searchIndexSampleData = new SearchIndexSampleData().withMessages(messages);
SearchIndex searchIndex1 = putSampleData(searchIndex.getId(), searchIndexSampleData, ADMIN_AUTH_HEADERS);
assertEquals(searchIndexSampleData, searchIndex1.getSampleData());
SearchIndex searchIndex2 = getSampleData(searchIndex.getId(), ADMIN_AUTH_HEADERS);
assertEquals(searchIndex2.getSampleData(), searchIndex1.getSampleData());
messages =
Arrays.asList(
"{\"email\": \"email1@email.com\", \"firstName\": \"Bob\", \"lastName\": \"Jones\"}",
"{\"email\": \"email2@email.com\", \"firstName\": \"Test\", \"lastName\": \"Jones\"}");
searchIndexSampleData.withMessages(messages);
SearchIndex putResponse = putSampleData(searchIndex2.getId(), searchIndexSampleData, ADMIN_AUTH_HEADERS);
assertEquals(searchIndexSampleData, putResponse.getSampleData());
searchIndex2 = getSampleData(searchIndex.getId(), ADMIN_AUTH_HEADERS);
assertEquals(searchIndexSampleData, searchIndex2.getSampleData());
}
@Test
void test_inheritDomain(TestInfo test) throws IOException {
// When domain is not set for a searchIndex, carry it forward from the search service
SearchServiceResourceTest serviceTest = new SearchServiceResourceTest();
CreateSearchService createService = serviceTest.createRequest(test).withDomain(DOMAIN.getFullyQualifiedName());
SearchService service = serviceTest.createEntity(createService, ADMIN_AUTH_HEADERS);
// Create a searchIndex without domain and ensure it inherits domain from the parent
CreateSearchIndex create = createRequest("user").withService(service.getFullyQualifiedName());
assertDomainInheritance(create, DOMAIN.getEntityReference());
}
@Override
public SearchIndex validateGetWithDifferentFields(SearchIndex searchIndex, boolean byName)
throws HttpResponseException {
// .../searchIndex?fields=owner
String fields = "";
searchIndex =
byName
? getSearchIndexByName(searchIndex.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getSearchIndex(searchIndex.getId(), fields, ADMIN_AUTH_HEADERS);
assertListNull(searchIndex.getOwner(), searchIndex.getFollowers(), searchIndex.getFollowers());
fields = "owner, followers, tags";
searchIndex =
byName
? getSearchIndexByName(searchIndex.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getSearchIndex(searchIndex.getId(), fields, ADMIN_AUTH_HEADERS);
assertListNotNull(searchIndex.getService(), searchIndex.getServiceType());
// Checks for other owner, tags, and followers is done in the base class
return searchIndex;
}
public SearchIndex getSearchIndex(UUID id, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource(id);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, SearchIndex.class, authHeaders);
}
public SearchIndex getSearchIndexByName(String fqn, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResourceByName(fqn);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, SearchIndex.class, authHeaders);
}
@Override
public CreateSearchIndex createRequest(String name) {
return new CreateSearchIndex()
.withName(name)
.withService(getContainer().getFullyQualifiedName())
.withFields(SEARCH_INDEX_FIELDS);
}
@Override
public EntityReference getContainer() {
return ELASTICSEARCH_SEARCH_SERVICE_REFERENCE;
}
@Override
public EntityReference getContainer(SearchIndex entity) {
return entity.getService();
}
@Override
public void validateCreatedEntity(
SearchIndex searchIndex, CreateSearchIndex createRequest, Map<String, String> authHeaders)
throws HttpResponseException {
assertReference(createRequest.getService(), searchIndex.getService());
// TODO add other fields
TestUtils.validateTags(createRequest.getTags(), searchIndex.getTags());
}
@Override
public void compareEntities(SearchIndex expected, SearchIndex updated, Map<String, String> authHeaders)
throws HttpResponseException {
assertReference(expected.getService(), expected.getService());
// TODO add other fields
TestUtils.validateTags(expected.getTags(), updated.getTags());
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (expected == actual) {
return;
}
assertCommonFieldChange(fieldName, expected, actual);
}
public SearchIndex putSampleData(UUID searchIndexId, SearchIndexSampleData data, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource(searchIndexId).path("/sampleData");
return TestUtils.put(target, data, SearchIndex.class, OK, authHeaders);
}
public SearchIndex getSampleData(UUID searchIndexId, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = getResource(searchIndexId).path("/sampleData");
return TestUtils.get(target, SearchIndex.class, authHeaders);
}
private static SearchIndexField getField(String name, SearchIndexDataType fieldDataType, TagLabel tag) {
List<TagLabel> tags = tag == null ? new ArrayList<>() : singletonList(tag);
return new SearchIndexField().withName(name).withDataType(fieldDataType).withDescription(name).withTags(tags);
}
private static void assertFields(List<SearchIndexField> expectedFields, List<SearchIndexField> actualFields)
throws HttpResponseException {
if (expectedFields == actualFields) {
return;
}
// Sort columns by name
assertEquals(expectedFields.size(), actualFields.size());
// Make a copy before sorting in case the lists are immutable
List<SearchIndexField> expected = new ArrayList<>(expectedFields);
List<SearchIndexField> actual = new ArrayList<>(actualFields);
expected.sort(Comparator.comparing(SearchIndexField::getName));
actual.sort(Comparator.comparing(SearchIndexField::getName));
for (int i = 0; i < expected.size(); i++) {
assertField(expected.get(i), actual.get(i));
}
}
private static void assertField(SearchIndexField expectedField, SearchIndexField actualField)
throws HttpResponseException {
assertNotNull(actualField.getFullyQualifiedName());
assertTrue(
expectedField.getName().equals(actualField.getName())
|| expectedField.getName().equals(actualField.getDisplayName()));
assertEquals(expectedField.getDescription(), actualField.getDescription());
assertEquals(expectedField.getDataType(), actualField.getDataType());
TestUtils.validateTags(expectedField.getTags(), actualField.getTags());
// Check the nested columns
assertFields(expectedField.getChildren(), actualField.getChildren());
}
}

208
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/SearchServiceResourceTest.java Normal file
View File

@ -0,0 +1,208 @@
package org.openmetadata.service.resources.services;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.OK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.service.util.EntityUtil.fieldAdded;
import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.assertResponse;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
import javax.ws.rs.client.WebTarget;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.schema.api.services.CreateSearchService;
import org.openmetadata.schema.entity.services.SearchService;
import org.openmetadata.schema.entity.services.connections.TestConnectionResult;
import org.openmetadata.schema.entity.services.connections.TestConnectionResultStatus;
import org.openmetadata.schema.services.connections.search.ElasticSearchConnection;
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.SearchConnection;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.services.searchIndexes.SearchServiceResource;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.TestUtils;
public class SearchServiceResourceTest extends EntityResourceTest<SearchService, CreateSearchService> {
public SearchServiceResourceTest() {
super(
Entity.SEARCH_SERVICE,
SearchService.class,
SearchServiceResource.SearchServiceList.class,
"services/searchServices",
"owner");
this.supportsPatch = false;
}
public void setupSearchService(TestInfo test) throws HttpResponseException {
SearchServiceResourceTest esSearchServiceResourceTest = new SearchServiceResourceTest();
CreateSearchService createSearchService =
esSearchServiceResourceTest
.createRequest(test, 1)
.withName("elasticSearch")
.withServiceType(CreateSearchService.SearchServiceType.ElasticSearch)
.withConnection(TestUtils.ELASTIC_SEARCH_CONNECTION);
SearchService esSearchService =
new SearchServiceResourceTest().createEntity(createSearchService, ADMIN_AUTH_HEADERS);
ELASTICSEARCH_SEARCH_SERVICE_REFERENCE = esSearchService.getEntityReference();
SearchServiceResourceTest osSearchServiceResourceTest = new SearchServiceResourceTest();
createSearchService =
osSearchServiceResourceTest
.createRequest(test, 1)
.withName("opensearch")
.withServiceType(CreateSearchService.SearchServiceType.OpenSearch)
.withConnection(TestUtils.OPEN_SEARCH_CONNECTION);
SearchService osSearchService =
new SearchServiceResourceTest().createEntity(createSearchService, ADMIN_AUTH_HEADERS);
OPENSEARCH_SEARCH_SERVICE_REFERENCE = osSearchService.getEntityReference();
}
@Test
void post_withoutRequiredFields_400_badRequest(TestInfo test) {
// Create StorageService with mandatory serviceType field empty
assertResponse(
() -> createEntity(createRequest(test).withServiceType(null), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[serviceType must not be null]");
// Create StorageService with mandatory connection field empty
assertResponse(
() -> createEntity(createRequest(test).withConnection(null), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[connection must not be null]");
}
@Test
void post_validService_as_admin_200_ok(TestInfo test) throws IOException {
// Create Storage service with different optional fields
Map<String, String> authHeaders = ADMIN_AUTH_HEADERS;
createAndCheckEntity(createRequest(test, 1).withDescription(null), authHeaders);
createAndCheckEntity(createRequest(test, 2).withDescription("description"), authHeaders);
createAndCheckEntity(createRequest(test, 3).withConnection(TestUtils.ELASTIC_SEARCH_CONNECTION), authHeaders);
}
@Test
void put_updateService_as_admin_2xx(TestInfo test) throws IOException {
SearchConnection connection1 =
new SearchConnection().withConfig(new ElasticSearchConnection().withHostPort("http://localhost:9300"));
SearchService service =
createAndCheckEntity(createRequest(test).withDescription(null).withConnection(connection1), ADMIN_AUTH_HEADERS);
ElasticSearchConnection credentials2 = new ElasticSearchConnection().withHostPort("https://localhost:9400");
SearchConnection connection2 = new SearchConnection().withConfig(credentials2);
// Update SearchService description and connection
CreateSearchService update = createRequest(test).withDescription("description1").withConnection(connection2);
ChangeDescription change = getChangeDescription(service.getVersion());
fieldAdded(change, "description", "description1");
fieldUpdated(change, "connection", connection1, connection2);
updateAndCheckEntity(update, OK, ADMIN_AUTH_HEADERS, TestUtils.UpdateType.MINOR_UPDATE, change);
}
@Test
void put_testConnectionResult_200(TestInfo test) throws IOException {
SearchService service = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS);
// By default, we have no result logged in
assertNull(service.getTestConnectionResult());
SearchService updatedService = putTestConnectionResult(service.getId(), TEST_CONNECTION_RESULT, ADMIN_AUTH_HEADERS);
// Validate that the data got properly stored
assertNotNull(updatedService.getTestConnectionResult());
assertEquals(TestConnectionResultStatus.SUCCESSFUL, updatedService.getTestConnectionResult().getStatus());
assertEquals(updatedService.getConnection(), service.getConnection());
// Check that the stored data is also correct
SearchService stored = getEntity(service.getId(), ADMIN_AUTH_HEADERS);
assertNotNull(stored.getTestConnectionResult());
assertEquals(TestConnectionResultStatus.SUCCESSFUL, stored.getTestConnectionResult().getStatus());
assertEquals(stored.getConnection(), service.getConnection());
}
public SearchService putTestConnectionResult(
UUID serviceId, TestConnectionResult testConnectionResult, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource(serviceId).path("/testConnectionResult");
return TestUtils.put(target, testConnectionResult, SearchService.class, OK, authHeaders);
}
@Override
public CreateSearchService createRequest(String name) {
return new CreateSearchService()
.withName(name)
.withServiceType(CreateSearchService.SearchServiceType.ElasticSearch)
.withConnection(
new SearchConnection().withConfig(new ElasticSearchConnection().withHostPort("http://localhost:9200")));
}
@Override
public void validateCreatedEntity(
SearchService service, CreateSearchService createRequest, Map<String, String> authHeaders)
throws HttpResponseException {
assertEquals(createRequest.getName(), service.getName());
SearchConnection expectedConnection = createRequest.getConnection();
SearchConnection actualConnection = service.getConnection();
validateConnection(expectedConnection, actualConnection, service.getServiceType());
}
@Override
public void compareEntities(SearchService expected, SearchService updated, Map<String, String> authHeaders)
throws HttpResponseException {
// PATCH operation is not supported by this entity
}
@Override
public SearchService validateGetWithDifferentFields(SearchService service, boolean byName)
throws HttpResponseException {
String fields = "";
service =
byName
? getEntityByName(service.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getEntity(service.getId(), fields, ADMIN_AUTH_HEADERS);
TestUtils.assertListNull(service.getOwner());
fields = "owner,tags";
service =
byName
? getEntityByName(service.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getEntity(service.getId(), fields, ADMIN_AUTH_HEADERS);
// Checks for other owner, tags, and followers is done in the base class
return service;
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
if (fieldName.equals("connection")) {
assertTrue(((String) actual).contains("-encrypted-value"));
} else {
super.assertCommonFieldChange(fieldName, expected, actual);
}
}
private void validateConnection(
SearchConnection expectedConnection,
SearchConnection actualConnection,
CreateSearchService.SearchServiceType serviceType) {
if (expectedConnection != null && actualConnection != null) {
if (serviceType == CreateSearchService.SearchServiceType.ElasticSearch) {
ElasticSearchConnection expectedESConnection = (ElasticSearchConnection) expectedConnection.getConfig();
ElasticSearchConnection actualESConnection;
if (actualConnection.getConfig() instanceof ElasticSearchConnection) {
actualESConnection = (ElasticSearchConnection) actualConnection.getConfig();
} else {
actualESConnection = JsonUtils.convertValue(actualConnection.getConfig(), ElasticSearchConnection.class);
}
assertEquals(expectedESConnection.getHostPort(), actualESConnection.getHostPort());
}
}
}
}

13
openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java
View File

@ -23,13 +23,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestMethodOrder;
import org.openmetadata.api.configuration.LogoConfiguration;
import org.openmetadata.schema.api.data.CreateContainer;
import org.openmetadata.schema.api.data.CreateDashboard;
import org.openmetadata.schema.api.data.CreateGlossary;
import org.openmetadata.schema.api.data.CreateGlossaryTerm;
import org.openmetadata.schema.api.data.CreatePipeline;
import org.openmetadata.schema.api.data.CreateTable;
import org.openmetadata.schema.api.data.CreateTopic;
import org.openmetadata.schema.api.data.*;
import org.openmetadata.schema.api.services.CreateDashboardService;
import org.openmetadata.schema.api.services.CreateDatabaseService;
import org.openmetadata.schema.api.services.CreateMessagingService;
@ -57,6 +51,7 @@ import org.openmetadata.service.resources.dqtests.TestSuiteResourceTest;
import org.openmetadata.service.resources.glossary.GlossaryResourceTest;
import org.openmetadata.service.resources.glossary.GlossaryTermResourceTest;
import org.openmetadata.service.resources.pipelines.PipelineResourceTest;
import org.openmetadata.service.resources.searchindex.SearchIndexResourceTest;
import org.openmetadata.service.resources.services.DashboardServiceResourceTest;
import org.openmetadata.service.resources.services.DatabaseServiceResourceTest;
import org.openmetadata.service.resources.services.MessagingServiceResourceTest;
@ -141,6 +136,10 @@ public class SystemResourceTest extends OpenMetadataApplicationTest {
CreateContainer createContainer = containerResourceTest.createRequest(test);
containerResourceTest.createEntity(createContainer, ADMIN_AUTH_HEADERS);
SearchIndexResourceTest SearchIndexResourceTest = new SearchIndexResourceTest();
CreateSearchIndex createSearchIndex = SearchIndexResourceTest.createRequest(test);
SearchIndexResourceTest.createEntity(createSearchIndex, ADMIN_AUTH_HEADERS);
GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest();
CreateGlossary createGlossary = glossaryResourceTest.createRequest(test);
glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS);

14
openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java
View File

@ -69,12 +69,15 @@ import org.openmetadata.schema.services.connections.metadata.AtlasConnection;
import org.openmetadata.schema.services.connections.mlmodel.MlflowConnection;
import org.openmetadata.schema.services.connections.pipeline.AirflowConnection;
import org.openmetadata.schema.services.connections.pipeline.GluePipelineConnection;
import org.openmetadata.schema.services.connections.search.ElasticSearchConnection;
import org.openmetadata.schema.services.connections.search.OpenSearchConnection;
import org.openmetadata.schema.services.connections.storage.S3Connection;
import org.openmetadata.schema.type.DashboardConnection;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.MessagingConnection;
import org.openmetadata.schema.type.MlModelConnection;
import org.openmetadata.schema.type.PipelineConnection;
import org.openmetadata.schema.type.SearchConnection;
import org.openmetadata.schema.type.StorageConnection;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.TagLabel.TagSource;
@ -111,6 +114,10 @@ public final class TestUtils {
public static final MlModelConnection MLFLOW_CONNECTION;
public static final StorageConnection S3_STORAGE_CONNECTION;
public static final SearchConnection ELASTIC_SEARCH_CONNECTION;
public static final SearchConnection OPEN_SEARCH_CONNECTION;
public static MetadataConnection AMUNDSEN_CONNECTION;
public static MetadataConnection ATLAS_CONNECTION;
@ -218,6 +225,13 @@ public final class TestUtils {
S3_STORAGE_CONNECTION = new StorageConnection().withConfig(new S3Connection().withAwsConfig(AWS_CREDENTIALS));
}
static {
ELASTIC_SEARCH_CONNECTION =
new SearchConnection().withConfig(new ElasticSearchConnection().withHostPort("http://localhost:9200"));
OPEN_SEARCH_CONNECTION =
new SearchConnection().withConfig(new OpenSearchConnection().withHostPort("http://localhost:9200"));
}
static {
try {
PIPELINE_URL = new URI("http://localhost:8080");

70
openmetadata-spec/src/main/resources/json/schema/api/data/createSearchIndex.json Normal file
View File

@ -0,0 +1,70 @@
{
"$id": "https://open-metadata.org/schema/api/data/createSearchIndex.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CreateSearchIndexRequest",
"description": "Create a SearchIndex entity request",
"type": "object",
"javaType": "org.openmetadata.schema.api.data.CreateSearchIndex",
"javaInterfaces": ["org.openmetadata.schema.CreateEntity"],
"properties": {
"name": {
"description": "Name that identifies this SearchIndex instance uniquely.",
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this SearchIndex.",
"type": "string"
},
"description": {
"description": "Description of the SearchIndex instance. What it has and how to use it.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"service": {
"description": "Fully qualified name of the search service where this searchIndex is hosted in",
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"fields": {
"description": "Fields in this SearchIndex.",
"type": "array",
"items": {
"$ref": "../../entity/data/searchIndex.json#/definitions/searchIndexField"
},
"default": null
},
"searchIndexSettings": {
"description": "Contains key/value pair of searchIndex settings.",
"$ref": "../../entity/data/searchIndex.json#/definitions/searchIndexSettings"
},
"owner": {
"description": "Owner of this SearchIndex",
"$ref": "../../type/entityReference.json"
},
"tags": {
"description": "Tags for this SearchIndex",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"
},
"domain" : {
"description": "Fully qualified name of the domain the SearchIndex belongs to.",
"type": "string",
"$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"dataProducts" : {
"description": "List of fully qualified names of data products this entity is part of.",
"type": "array",
"items" : {
"$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName"
}
}
},
"required": ["name", "service", "fields"],
"additionalProperties": false
}

48
openmetadata-spec/src/main/resources/json/schema/api/services/createSearchService.json Normal file
View File

@ -0,0 +1,48 @@
{
"$id": "https://open-metadata.org/schema/api/services/createSearchService.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CreateSearchServiceRequest",
"description": "Create Search Service entity request",
"type": "object",
"javaType": "org.openmetadata.schema.api.services.CreateSearchService",
"javaInterfaces": ["org.openmetadata.schema.CreateEntity"],
"properties": {
"name": {
"description": "Name that identifies the this entity instance uniquely",
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this search service. It could be title or label from the source services.",
"type": "string"
},
"description": {
"description": "Description of search service entity.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"serviceType": {
"$ref": "../../entity/services/searchService.json#/definitions/searchServiceType"
},
"connection": {
"$ref": "../../entity/services/searchService.json#/definitions/searchConnection"
},
"tags": {
"description": "Tags for this Search Service.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"owner": {
"description": "Owner of this search service.",
"$ref": "../../type/entityReference.json"
},
"domain" : {
"description": "Fully qualified name of the domain the Search Service belongs to.",
"type": "string"
}
},
"required": ["name", "serviceType", "connection"],
"additionalProperties": false
}

229
openmetadata-spec/src/main/resources/json/schema/entity/data/searchIndex.json Normal file
View File

@ -0,0 +1,229 @@
{
"$id": "https://open-metadata.org/schema/entity/data/SearchIndex.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "SearchIndex",
"$comment": "@om-entity-type",
"description": "A `SearchIndex` is a index mapping definition in ElasticSearch or OpenSearch",
"type": "object",
"javaType": "org.openmetadata.schema.entity.data.SearchIndex",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"searchIndexSettings": {
"javaType": "org.openmetadata.schema.type.searchindex.SearchIndexSettings",
"description": "Contains key/value pair of SearchIndex Settings.",
"type": "object"
},
"searchIndexSampleData": {
"type": "object",
"javaType": "org.openmetadata.schema.type.searchindex.SearchIndexSampleData",
"description": "This schema defines the type to capture sample data for a SearchIndex.",
"properties": {
"messages": {
"description": "List of local sample messages for a SearchIndex.",
"type": "array",
"items": {
"type": "string"
}
}
},
"additionalProperties": false
},
"dataType": {
"javaType": "org.openmetadata.schema.type.SearchIndexDataType",
"description": "This enum defines the type of data stored in a searchIndex.",
"type": "string",
"enum": [
"NUMBER",
"TEXT",
"BINARY",
"TIMESTAMP",
"TIMESTAMPZ",
"TIME",
"DATE",
"DATETIME",
"KEYWORD",
"ARRAY",
"OBJECT",
"FLATTENED",
"NESTED",
"JOIN",
"RANGE",
"IP",
"VERSION",
"MURMUR3",
"AGGREGATE_METRIC_DOUBLE",
"HISTOGRAM",
"ANNOTATED-TEXT",
"COMPLETION",
"SEARCH_AS_YOU_TYPE",
"DENSE_VECTOR",
"RANK_FEATURE",
"RANK_FEATURES",
"GEO_POINT",
"GEO_SHAPE",
"POINT",
"SHAPE",
"PERCOLATOR"
]
},
"searchIndexFieldName": {
"description": "Local name (not fully qualified name) of the field. ",
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^((?!::).)*$"
},
"searchIndexField": {
"type": "object",
"javaType": "org.openmetadata.schema.type.SearchIndexField",
"description": "This schema defines the type for a field in a searchIndex.",
"properties": {
"name": {
"$ref": "#/definitions/searchIndexFieldName"
},
"displayName": {
"description": "Display Name that identifies this searchIndexField name.",
"type": "string"
},
"dataType": {
"description": "Data type of the searchIndex (int, date etc.).",
"$ref": "#/definitions/dataType"
},
"dataTypeDisplay": {
"description": "Display name used for dataType. ",
"type": "string"
},
"description": {
"description": "Description of the field.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"fullyQualifiedName": {
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"tags": {
"description": "Tags associated with the column.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"children": {
"description": "Child columns if dataType has properties.",
"type": "array",
"items": {
"$ref": "#/definitions/searchIndexField"
},
"default": null
}
},
"required": [
"name",
"dataType"
],
"additionalProperties": false
}
},
"properties": {
"id": {
"description": "Unique identifier that identifies this SearchIndex instance.",
"$ref": "../../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Name that identifies the SearchIndex.",
"$ref": "../../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "Name that uniquely identifies a SearchIndex in the format 'searchServiceName.searchIndexName'.",
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"displayName": {
"description": "Display Name that identifies this SearchIndex. It could be title or label from the source services.",
"type": "string"
},
"description": {
"description": "Description of the SearchIndex instance.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"version": {
"description": "Metadata version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"service": {
"description": "Link to the search cluster/service where this SearchIndex is hosted in.",
"$ref": "../../type/entityReference.json"
},
"serviceType": {
"description": "Service type where this SearchIndex is hosted in.",
"$ref": "../services/searchService.json#/definitions/searchServiceType"
},
"fields": {
"description": "Fields in this SearchIndex.",
"type": "array",
"items": {
"$ref": "#/definitions/searchIndexField"
},
"default": null
},
"searchIndexSettings": {
"description": "Contains key/value pair of searchIndex settings.",
"$ref": "#/definitions/searchIndexSettings"
},
"sampleData": {
"description": "Sample data for a searchIndex.",
"$ref": "#/definitions/searchIndexSampleData",
"default": null
},
"owner": {
"description": "Owner of this searchIndex.",
"$ref": "../../type/entityReference.json"
},
"followers": {
"description": "Followers of this searchIndex.",
"$ref": "../../type/entityReferenceList.json"
},
"tags": {
"description": "Tags for this searchIndex.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"href": {
"description": "Link to the resource corresponding to this entity.",
"$ref": "../../type/basic.json#/definitions/href"
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"
},
"domain" : {
"description": "Domain the SearchIndex belongs to. When not set, the SearchIndex inherits the domain from the messaging service it belongs to.",
"$ref": "../../type/entityReference.json"
},
"dataProducts" : {
"description": "List of of data products this entity is part of.",
"$ref" : "../../type/entityReferenceList.json"
}
},
"required": ["id", "name", "service", "fields"],
"additionalProperties": false
}

36
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/customSearchConnection.json Normal file
View File

@ -0,0 +1,36 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/search/customSearchConnection.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "CustomSearchConnection",
"description": "Custom Search Service connection to build a source that is not supported by OpenMetadata yet.",
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.search.CustomSearchConnection",
"definitions": {
"customSearchType": {
"title": "Service Type",
"description": "Custom search service type",
"type": "string",
"enum": ["CustomSearch"],
"default": "CustomSearch"
}
},
"properties": {
"type": {
"title": "Service Type",
"description": "Custom search service type",
"$ref": "#/definitions/customSearchType",
"default": "CustomSearch"
},
"sourcePythonClass": {
"title": "Source Python Class Name",
"description": "Source Python Class Name to instantiated by the ingestion workflow",
"type": "string"
},
"connectionOptions": {
"title": "Connection Options",
"$ref": "../connectionBasicType.json#/definitions/connectionOptions"
}
},
"additionalProperties": false,
"required": ["type"]
}

80
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json Normal file
View File

@ -0,0 +1,80 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/search/elasticSearchConnection.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ElasticSearch Connection",
"description": "ElasticSearch Connection.",
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.search.ElasticSearchConnection",
"definitions": {
"elasticSearchType": {
"description": "ElasticSearch service type",
"type": "string",
"enum": ["ElasticSearch"],
"default": "ElasticSearch"
}
},
"properties": {
"type": {
"title": "ElasticSearch Type",
"description": "ElasticSearch Type",
"$ref": "#/definitions/elasticSearchType",
"default": "ElasticSearch"
},
"hostPort": {
"title": "Host and Port",
"description": "Host and port of the ElasticSearch service.",
"type": "string"
},
"scheme": {
"description": "Http/Https connection scheme",
"type": "string",
"default": "http"
},
"username": {
"description": "Elastic Search Username for Login",
"type": "string"
},
"password": {
"description": "Elastic Search Password for Login",
"type": "string"
},
"truststorePath": {
"description": "Truststore Path",
"type": "string"
},
"truststorePassword": {
"description": "Truststore Password",
"type": "string"
},
"connectionTimeoutSecs": {
"description": "Connection Timeout in Seconds",
"type": "integer",
"default": 5
},
"socketTimeoutSecs": {
"description": "Socket Timeout in Seconds",
"type": "integer",
"default": 60
},
"keepAliveTimeoutSecs": {
"description": "Keep Alive Timeout in Seconds",
"type": "integer"
},
"connectionOptions": {
"title": "Connection Options",
"$ref": "../connectionBasicType.json#/definitions/connectionOptions"
},
"connectionArguments": {
"title": "Connection Arguments",
"$ref": "../connectionBasicType.json#/definitions/connectionArguments"
},
"supportsMetadataExtraction": {
"title": "Supports Metadata Extraction",
"$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
}
},
"additionalProperties": false,
"required": [
"hostPort"
]
}

79
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/openSearchConnection.json Normal file
View File

@ -0,0 +1,79 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/search/openSearchConnection.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "OpenSearch Connection",
"description": "OpenSearch Connection.",
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.search.OpenSearchConnection",
"definitions": {
"openSearchType": {
"description": "OpenSearch service type",
"type": "string",
"enum": ["OpenSearch"],
"default": "OpenSearch"
}
},
"properties": {
"type": {
"title": "Service Type",
"description": "Service Type",
"$ref": "#/definitions/openSearchType",
"default": "OpenSearch"
},
"hostPort": {
"title": "Host and Port",
"description": "Host and port of the OpenSearch service.",
"type": "string"
},
"scheme": {
"description": "Http/Https connection scheme",
"type": "string"
},
"username": {
"description": "OpenSearch Username for Login",
"type": "string"
},
"password": {
"description": "OpenSearch Password for Login",
"type": "string"
},
"truststorePath": {
"description": "Truststore Path",
"type": "string"
},
"truststorePassword": {
"description": "Truststore Password",
"type": "string"
},
"connectionTimeoutSecs": {
"description": "Connection Timeout in Seconds",
"type": "integer",
"default": 5
},
"socketTimeoutSecs": {
"description": "Socket Timeout in Seconds",
"type": "integer",
"default": 60
},
"keepAliveTimeoutSecs": {
"description": "Keep Alive Timeout in Seconds",
"type": "integer"
},
"connectionOptions": {
"title": "Connection Options",
"$ref": "../connectionBasicType.json#/definitions/connectionOptions"
},
"connectionArguments": {
"title": "Connection Arguments",
"$ref": "../connectionBasicType.json#/definitions/connectionArguments"
},
"supportsMetadataExtraction": {
"title": "Supports Metadata Extraction",
"$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
}
},
"additionalProperties": false,
"required": [
"hostPort"
]
}

3
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json
View File

@ -28,6 +28,9 @@
},
{
"$ref": "../storageService.json#/definitions/storageConnection"
},
{
"$ref": "../searchService.json#/definitions/searchConnection"
}
]
}

142
openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json Normal file
View File

@ -0,0 +1,142 @@
{
"$id": "https://open-metadata.org/schema/entity/services/searchService.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Search Service",
"description": "This schema defines the Search Service entity, such as ElasticSearch, OpenSearch.",
"type": "object",
"javaType": "org.openmetadata.schema.entity.services.SearchService",
"javaInterfaces": [
"org.openmetadata.schema.EntityInterface",
"org.openmetadata.schema.ServiceEntityInterface"
],
"definitions": {
"searchServiceType": {
"description": "Type of search service such as ElasticSearch or OpenSearch.",
"javaInterfaces": [
"org.openmetadata.schema.EnumInterface"
],
"type": "string",
"enum": [
"ElasticSearch",
"OpenSearch",
"CustomSearch"
],
"javaEnums": [
{
"name": "ElasticSearch"
},
{
"name": "OpenSearch"
},
{
"name": "CustomSearch"
}
]
},
"searchConnection": {
"type": "object",
"javaType": "org.openmetadata.schema.type.SearchConnection",
"description": "search Connection.",
"javaInterfaces": [
"org.openmetadata.schema.ServiceConnectionEntityInterface"
],
"properties": {
"config": {
"mask": true,
"oneOf": [
{
"$ref": "connections/search/elasticSearchConnection.json"
},
{
"$ref": "connections/search/openSearchConnection.json"
},
{
"$ref": "connections/search/customSearchConnection.json"
}
]
}
},
"additionalProperties": false
}
},
"properties": {
"id": {
"description": "Unique identifier of this search service instance.",
"$ref": "../../type/basic.json#/definitions/uuid"
},
"name": {
"description": "Name that identifies this search service.",
"$ref": "../../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "FullyQualifiedName same as `name`.",
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
},
"displayName": {
"description": "Display Name that identifies this search service.",
"type": "string"
},
"serviceType": {
"description": "Type of search service such as S3, GCS, AZURE...",
"$ref": "#/definitions/searchServiceType"
},
"description": {
"description": "Description of a search service instance.",
"$ref": "../../type/basic.json#/definitions/markdown"
},
"connection": {
"$ref": "#/definitions/searchConnection"
},
"pipelines": {
"description": "References to pipelines deployed for this search service to extract metadata etc..",
"$ref": "../../type/entityReferenceList.json"
},
"testConnectionResult": {
"description": "Last test connection results for this service",
"$ref": "connections/testConnectionResult.json"
},
"tags": {
"description": "Tags for this search Service.",
"type": "array",
"items": {
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"version": {
"description": "Metadata version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/entityVersion"
},
"updatedAt": {
"description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
"$ref": "../../type/basic.json#/definitions/timestamp"
},
"updatedBy": {
"description": "User who made the update.",
"type": "string"
},
"href": {
"description": "Link to the resource corresponding to this search service.",
"$ref": "../../type/basic.json#/definitions/href"
},
"owner": {
"description": "Owner of this search service.",
"$ref": "../../type/entityReference.json"
},
"changeDescription": {
"description": "Change that lead to this version of the entity.",
"$ref": "../../type/entityHistory.json#/definitions/changeDescription"
},
"deleted": {
"description": "When `true` indicates the entity has been soft deleted.",
"type": "boolean",
"default": false
},
"domain" : {
"description": "Domain the search service belongs to.",
"$ref": "../../type/entityReference.json"
}
},
"required": ["id", "name", "serviceType"],
"additionalProperties": false
}

3
openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json
View File

@ -12,7 +12,8 @@
"Metadata",
"MlModel",
"Pipeline",
"Storage"
"Storage",
"Search"
],
"additionalProperties": false
}

27
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json Normal file
View File

@ -0,0 +1,27 @@
{
"$id": "https://open-metadata.org/schema/metadataIngestion/searchServiceMetadataPipeline.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "SearchServiceMetadataPipeline",
"description": "SearchService Metadata Pipeline Configuration.",
"type": "object",
"definitions": {
"searchMetadataConfigType": {
"description": "Search Source Config Metadata Pipeline type",
"type": "string",
"enum": ["SearchMetadata"],
"default": "SearchMetadata"
}
},
"properties": {
"type": {
"description": "Pipeline type",
"$ref": "#/definitions/searchMetadataConfigType",
"default": "SearchMetadata"
},
"searchIndexFilterPattern": {
"description": "Regex to only fetch search indexes that matches the pattern.",
"$ref": "../type/filterPattern.json#/definitions/filterPattern"
}
},
"additionalProperties": false
}

3
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json
View File

@ -39,6 +39,9 @@
{
"$ref": "storageServiceMetadataPipeline.json"
},
{
"$ref": "searchServiceMetadataPipeline.json"
},
{
"$ref": "testSuitePipeline.json"
},