yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-17 03:11:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Issues in Search Relevancy (#21841)" (#21853)

Browse Source 
This reverts commit f388e570c1dac5b9eee31364870fb66e42715f18.
This commit is contained in:
Sriharsha Chintalapani 2025-06-18 16:43:34 -07:00 committed by GitHub
parent f388e570c1
commit 8adda4955c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 45 additions and 571 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
ingestion/examples/sample_data/datasets/tables.json
View File

@ -32491,218 +32491,6 @@
}, },
"sourceUrl": "http://localhost:8080/test_long_description_table", "sourceUrl": "http://localhost:8080/test_long_description_table",
"tags": [] "tags": []
},
{
"name": "ssot_utilization_detail",
"description": "A detail of each agent's time throughout the day. Individual events in each system are collapsed by aux_code, aux_channel, worktype_name, and reporting_date.",
"version": 0.5,
"updatedAt": 1638354087391,
"updatedBy": "anonymous",
"tableType": "Regular",
"columns": [
{
"name": "reporting_date",
"dataType": "DATE",
"dataTypeDisplay": "date",
"description": "The date the event began.",
"tags": [],
"ordinalPosition": 1
},
{
"name": "agent_id",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar",
"description": "The unique identifier for the agent from customer_success.remitly_employee.",
"tags": [],
"ordinalPosition": 2
},
{
"name": "agent_email",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar",
"description": "The agent email.",
"tags": [],
"ordinalPosition": 3
}
],
"database": {
"id": "50da1ff8-4e1d-4967-8931-45edbf4dd908",
"type": "database",
"name": "sample_data.ecommerce_db",
"description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.",
"href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908"
},
"tags": [],
"usageSummary": {
"dailyStats": {
"count": 0,
"percentileRank": 0
},
"weeklyStats": {
"count": 0,
"percentileRank": 0
},
"monthlyStats": {
"count": 0,
"percentileRank": 0
},
"date": "2021-12-01"
},
"followers": []
},
{
"name": "itm_utilization_detail_login_events",
"description": "An intermediate table used to create ssot_utilization_detail. This table cleans up the \"CS Platform: login\" event dimensions to be congruent with downstream analytics.",
"version": 0.3,
"updatedAt": 1638354087391,
"updatedBy": "anonymous",
"tableType": "Regular",
"columns": [
{
"name": "reporting_date",
"dataType": "DATE",
"dataTypeDisplay": "date",
"description": "Event date",
"tags": [],
"ordinalPosition": 1
},
{
"name": "agent_id",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar",
"description": "Agent identifier",
"tags": [],
"ordinalPosition": 2
}
],
"database": {
"id": "50da1ff8-4e1d-4967-8931-45edbf4dd908",
"type": "database",
"name": "sample_data.ecommerce_db",
"description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.",
"href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908"
},
"tags": [],
"usageSummary": {
"dailyStats": {
"count": 10,
"percentileRank": 30
},
"weeklyStats": {
"count": 50,
"percentileRank": 40
},
"monthlyStats": {
"count": 200,
"percentileRank": 50
},
"date": "2021-12-01"
},
"followers": []
},
{
"name": "customer_metrics_daily",
"description": "Daily customer metrics aggregated from various sources including ssot_utilization_detail and other operational tables.",
"version": 0.2,
"updatedAt": 1638354087391,
"updatedBy": "anonymous",
"tableType": "Regular",
"columns": [
{
"name": "metric_date",
"dataType": "DATE",
"dataTypeDisplay": "date",
"description": "Date of the metric",
"tags": [],
"ordinalPosition": 1
},
{
"name": "customer_count",
"dataType": "INT",
"dataTypeDisplay": "integer",
"description": "Total number of customers",
"tags": [],
"ordinalPosition": 2
}
],
"database": {
"id": "50da1ff8-4e1d-4967-8931-45edbf4dd908",
"type": "database",
"name": "sample_data.ecommerce_db",
"description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.",
"href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908"
},
"tags": [],
"usageSummary": {
"dailyStats": {
"count": 100,
"percentileRank": 70
},
"weeklyStats": {
"count": 500,
"percentileRank": 80
},
"monthlyStats": {
"count": 2000,
"percentileRank": 90
},
"date": "2021-12-01"
},
"followers": []
},
{
"name": "agent_performance_summary",
"description": "Summary of agent performance metrics derived from multiple tables including ssot_utilization_detail for comprehensive reporting.",
"version": 0.4,
"updatedAt": 1638354087391,
"updatedBy": "anonymous",
"tableType": "Regular",
"columns": [
{
"name": "agent_id",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar",
"description": "Agent identifier",
"tags": [],
"ordinalPosition": 1
},
{
"name": "performance_score",
"dataType": "DECIMAL",
"dataTypeDisplay": "decimal",
"description": "Overall performance score",
"tags": [],
"ordinalPosition": 2
}
],
"database": {
"id": "50da1ff8-4e1d-4967-8931-45edbf4dd908",
"type": "database",
"name": "sample_data.ecommerce_db",
"description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.",
"href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908"
},
"tags": [],
"usageSummary": {
"dailyStats": {
"count": 50,
"percentileRank": 60
},
"weeklyStats": {
"count": 250,
"percentileRank": 65
},
"monthlyStats": {
"count": 1000,
"percentileRank": 70
},
"date": "2021-12-01"
},
"followers": []
} }
] ]
} }

6
openmetadata-service/pom.xml
View File

@ -948,12 +948,6 @@
<artifactId>opensearch-rest-high-level-client</artifactId> <artifactId>opensearch-rest-high-level-client</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.open-metadata</groupId>
<artifactId>opensearch-deps</artifactId>
<version>1.8.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies> </dependencies>
<profiles> <profiles>

19
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchSourceBuilderFactory.java
View File

@ -72,20 +72,6 @@ public interface SearchSourceBuilderFactory<S, Q, H, F> {
* @return a search source builder configured for the specific entity type * @return a search source builder configured for the specific entity type
*/ */
default S getSearchSourceBuilder(String index, String q, int from, int size) { default S getSearchSourceBuilder(String index, String q, int from, int size) {
return getSearchSourceBuilder(index, q, from, size, false);
}
/**
* Get the appropriate search source builder based on the index name.
*
* @param index the index name
* @param q the search query
* @param from the starting offset
* @param size the number of results to return
* @param explain whether to include explanation of the search results
* @return a search source builder configured for the specific entity type
*/
default S getSearchSourceBuilder(String index, String q, int from, int size, boolean explain) {
String indexName = Entity.getSearchRepository().getIndexNameWithoutAlias(index); String indexName = Entity.getSearchRepository().getIndexNameWithoutAlias(index);
if (isTimeSeriesIndex(indexName)) { if (isTimeSeriesIndex(indexName)) {
@ -101,7 +87,7 @@ public interface SearchSourceBuilderFactory<S, Q, H, F> {
} }
if (isDataAssetIndex(indexName)) { if (isDataAssetIndex(indexName)) {
return buildDataAssetSearchBuilder(indexName, q, from, size, explain); return buildDataAssetSearchBuilder(indexName, q, from, size);
} }
if (indexName.equals("all") || indexName.equals("dataAsset")) { if (indexName.equals("all") || indexName.equals("dataAsset")) {
@ -119,9 +105,6 @@ public interface SearchSourceBuilderFactory<S, Q, H, F> {
S buildDataAssetSearchBuilder(String indexName, String query, int from, int size); S buildDataAssetSearchBuilder(String indexName, String query, int from, int size);
S buildDataAssetSearchBuilder(
String indexName, String query, int from, int size, boolean explain);
S buildCommonSearchBuilder(String query, int from, int size); S buildCommonSearchBuilder(String query, int from, int size);
S buildUserOrTeamSearchBuilder(String query, int from, int size); S buildUserOrTeamSearchBuilder(String query, int from, int size);

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -366,11 +366,7 @@ public class ElasticSearchClient implements SearchClient {
new ElasticSearchSourceBuilderFactory(searchSettings); new ElasticSearchSourceBuilderFactory(searchSettings);
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder searchSourceBuilder =
searchBuilderFactory.getSearchSourceBuilder( searchBuilderFactory.getSearchSourceBuilder(
request.getIndex(), request.getIndex(), request.getQuery(), request.getFrom(), request.getSize());
request.getQuery(),
request.getFrom(),
request.getSize(),
request.getExplain());
buildSearchRBACQuery(subjectContext, searchSourceBuilder); buildSearchRBACQuery(subjectContext, searchSourceBuilder);
// Add Filter // Add Filter

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java
View File

@ -163,12 +163,6 @@ public class ElasticSearchSourceBuilderFactory
@Override @Override
public SearchSourceBuilder buildDataAssetSearchBuilder( public SearchSourceBuilder buildDataAssetSearchBuilder(
String indexName, String query, int from, int size) { String indexName, String query, int from, int size) {
return buildDataAssetSearchBuilder(indexName, query, from, size, false);
}
@Override
public SearchSourceBuilder buildDataAssetSearchBuilder(
String indexName, String query, int from, int size, boolean explain) {
AssetTypeConfiguration assetConfig = findAssetTypeConfig(indexName, searchSettings); AssetTypeConfiguration assetConfig = findAssetTypeConfig(indexName, searchSettings);
Map<String, Float> fuzzyFields; Map<String, Float> fuzzyFields;
Map<String, Float> nonFuzzyFields; Map<String, Float> nonFuzzyFields;
@ -318,7 +312,7 @@ public class ElasticSearchSourceBuilderFactory
} }
addConfiguredAggregations(searchSourceBuilder, assetConfig); addConfiguredAggregations(searchSourceBuilder, assetConfig);
searchSourceBuilder.explain(explain); searchSourceBuilder.explain(true);
return searchSourceBuilder; return searchSourceBuilder;
} }

29
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java
View File

@ -19,11 +19,11 @@ import static org.openmetadata.service.util.FullyQualifiedName.getParentFQN;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.EntityInterface;
@ -128,12 +128,29 @@ public interface SearchIndex {
} }
default Set<String> getFQNParts(String fqn) { default Set<String> getFQNParts(String fqn) {
var parts = FullyQualifiedName.split(fqn); Set<String> fqnParts = new HashSet<>();
var entityName = parts[parts.length - 1]; fqnParts.add(fqn);
String parent = FullyQualifiedName.getParentFQN(fqn);
while (parent != null) {
fqnParts.add(parent);
parent = FullyQualifiedName.getParentFQN(parent);
}
return fqnParts;
}
return FullyQualifiedName.getAllParts(fqn).stream() // Add suggest inputs to fqnParts to support partial/wildcard search on names.
.filter(part -> !part.equals(entityName)) // In some case of basic Test suite name is not part of the fullyQualifiedName, so it must be
.collect(Collectors.toSet()); // added separately.
default Set<String> getFQNParts(String fqn, List<String> fqnSplits) {
Set<String> fqnParts = new HashSet<>();
fqnParts.add(fqn);
String parent = FullyQualifiedName.getParentFQN(fqn);
while (parent != null) {
fqnParts.add(parent);
parent = FullyQualifiedName.getParentFQN(parent);
}
fqnParts.addAll(fqnSplits);
return fqnParts;
} }
default List<EntityReference> getEntitiesWithDisplayName(List<EntityReference> entities) { default List<EntityReference> getEntitiesWithDisplayName(List<EntityReference> entities) {

15
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -137,7 +137,6 @@ import os.org.opensearch.action.delete.DeleteRequest;
import os.org.opensearch.action.get.GetRequest; import os.org.opensearch.action.get.GetRequest;
import os.org.opensearch.action.get.GetResponse; import os.org.opensearch.action.get.GetResponse;
import os.org.opensearch.action.search.SearchResponse; import os.org.opensearch.action.search.SearchResponse;
import os.org.opensearch.action.search.SearchType;
import os.org.opensearch.action.support.WriteRequest; import os.org.opensearch.action.support.WriteRequest;
import os.org.opensearch.action.support.master.AcknowledgedResponse; import os.org.opensearch.action.support.master.AcknowledgedResponse;
import os.org.opensearch.action.update.UpdateRequest; import os.org.opensearch.action.update.UpdateRequest;
@ -385,11 +384,7 @@ public class OpenSearchClient implements SearchClient {
new OpenSearchSourceBuilderFactory(searchSettings); new OpenSearchSourceBuilderFactory(searchSettings);
SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder searchSourceBuilder =
searchBuilderFactory.getSearchSourceBuilder( searchBuilderFactory.getSearchSourceBuilder(
request.getIndex(), request.getIndex(), request.getQuery(), request.getFrom(), request.getSize());
request.getQuery(),
request.getFrom(),
request.getSize(),
request.getExplain());
buildSearchRBACQuery(subjectContext, searchSourceBuilder); buildSearchRBACQuery(subjectContext, searchSourceBuilder);
@ -514,10 +509,6 @@ public class OpenSearchClient implements SearchClient {
os.org.opensearch.action.search.SearchRequest searchRequest = os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(request.getIndex()); new os.org.opensearch.action.search.SearchRequest(request.getIndex());
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
// Use DFS Query Then Fetch for consistent scoring across shards
searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
os.org.opensearch.action.search.SearchResponse response = os.org.opensearch.action.search.SearchResponse response =
client.search(searchRequest, os.org.opensearch.client.RequestOptions.DEFAULT); client.search(searchRequest, os.org.opensearch.client.RequestOptions.DEFAULT);
if (response.getHits() != null if (response.getHits() != null
@ -550,10 +541,6 @@ public class OpenSearchClient implements SearchClient {
os.org.opensearch.action.search.SearchRequest osRequest = os.org.opensearch.action.search.SearchRequest osRequest =
new os.org.opensearch.action.search.SearchRequest(request.getIndex()); new os.org.opensearch.action.search.SearchRequest(request.getIndex());
osRequest.source(searchSourceBuilder); osRequest.source(searchSourceBuilder);
// Use DFS Query Then Fetch for consistent scoring across shards
osRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
getSearchBuilderFactory().addAggregationsToNLQQuery(searchSourceBuilder, request.getIndex()); getSearchBuilderFactory().addAggregationsToNLQQuery(searchSourceBuilder, request.getIndex());
SearchResponse searchResponse = client.search(osRequest, OPENSEARCH_REQUEST_OPTIONS); SearchResponse searchResponse = client.search(osRequest, OPENSEARCH_REQUEST_OPTIONS);
return Response.status(Response.Status.OK).entity(searchResponse.toString()).build(); return Response.status(Response.Status.OK).entity(searchResponse.toString()).build();

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java
View File

@ -162,12 +162,6 @@ public class OpenSearchSourceBuilderFactory
@Override @Override
public SearchSourceBuilder buildDataAssetSearchBuilder( public SearchSourceBuilder buildDataAssetSearchBuilder(
String indexName, String query, int from, int size) { String indexName, String query, int from, int size) {
return buildDataAssetSearchBuilder(indexName, query, from, size, false);
}
@Override
public SearchSourceBuilder buildDataAssetSearchBuilder(
String indexName, String query, int from, int size, boolean explain) {
AssetTypeConfiguration assetConfig = findAssetTypeConfig(indexName, searchSettings); AssetTypeConfiguration assetConfig = findAssetTypeConfig(indexName, searchSettings);
Map<String, Float> fuzzyFields; Map<String, Float> fuzzyFields;
Map<String, Float> nonFuzzyFields; Map<String, Float> nonFuzzyFields;
@ -318,7 +312,7 @@ public class OpenSearchSourceBuilderFactory
} }
addConfiguredAggregations(searchSourceBuilder, assetConfig); addConfiguredAggregations(searchSourceBuilder, assetConfig);
searchSourceBuilder.explain(explain); searchSourceBuilder.explain(true);
return searchSourceBuilder; return searchSourceBuilder;
} }

61
openmetadata-service/src/main/java/org/openmetadata/service/util/FullyQualifiedName.java
View File

@ -14,9 +14,7 @@
package org.openmetadata.service.util; package org.openmetadata.service.util;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.BailErrorStrategy;
@ -208,63 +206,4 @@ public class FullyQualifiedName {
public static String getColumnName(String columnFQN) { public static String getColumnName(String columnFQN) {
return FullyQualifiedName.split(columnFQN)[4]; // Get from column name from FQN return FullyQualifiedName.split(columnFQN)[4]; // Get from column name from FQN
} }
/**
* Generates all possible FQN parts for search and matching purposes.
* For example, given FQN "service.database.schema.table", this method generates:
* - Full hierarchy: "service", "service.database", "service.database.schema", "service.database.schema.table"
* - Individual parts: "service", "database", "schema", "table"
* - Bottom-up combinations: "database.schema.table", "schema.table", "table"
*
* @param fqn The fully qualified name to generate parts from
* @return Set of all possible FQN parts
*/
public static Set<String> getAllParts(String fqn) {
var parts = split(fqn);
var fqnParts = new HashSet<String>();
// Generate all possible sub-paths
for (int start = 0; start < parts.length; start++) {
for (int end = start + 1; end <= parts.length; end++) {
var subPath =
String.join(Entity.SEPARATOR, java.util.Arrays.copyOfRange(parts, start, end));
fqnParts.add(subPath);
}
}
return fqnParts;
}
/**
* Generates hierarchical FQN parts from root to the full FQN.
* For example, given FQN "service.database.schema.table", this method generates:
* ["service", "service.database", "service.database.schema", "service.database.schema.table"]
*
* @param fqn The fully qualified name to generate hierarchy from
* @return List of hierarchical FQN parts from root to full FQN
*/
public static List<String> getHierarchicalParts(String fqn) {
var parts = split(fqn);
return java.util.stream.IntStream.rangeClosed(1, parts.length)
.mapToObj(i -> String.join(Entity.SEPARATOR, java.util.Arrays.copyOfRange(parts, 0, i)))
.toList();
}
/**
* Gets all ancestor FQNs for a given FQN.
* For example, given FQN "service.database.schema.table", this method returns:
* ["service.database.schema", "service.database", "service"]
*
* @param fqn The fully qualified name to get ancestors from
* @return List of ancestor FQNs (excluding the input FQN itself)
*/
public static List<String> getAncestors(String fqn) {
var parts = split(fqn);
return java.util.stream.IntStream.range(1, parts.length)
.mapToObj(
i ->
String.join(
Entity.SEPARATOR, java.util.Arrays.copyOfRange(parts, 0, parts.length - i)))
.toList();
}
} }

34
openmetadata-service/src/main/resources/json/data/searchSettings/searchSettings.json
View File

@ -154,7 +154,7 @@
"aggregations": [ "aggregations": [
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum" "boostMode": "multiply"
}, },
{ {
"assetType": "databaseSchema", "assetType": "databaseSchema",
@ -200,7 +200,7 @@
"aggregations": [ "aggregations": [
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum" "boostMode": "multiply"
}, },
{ {
"assetType": "table", "assetType": "table",
@ -271,7 +271,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"description", "description",
@ -341,7 +341,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"description", "description",
@ -400,7 +400,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"description", "description",
@ -468,7 +468,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -571,7 +571,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -641,7 +641,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -701,7 +701,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -762,7 +762,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -823,7 +823,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -888,7 +888,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -976,7 +976,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -1027,7 +1027,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -1073,7 +1073,7 @@
"aggregations": [ "aggregations": [
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -1127,7 +1127,7 @@
"aggregations": [ "aggregations": [
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",
@ -1204,7 +1204,7 @@
} }
], ],
"scoreMode": "sum", "scoreMode": "sum",
"boostMode": "sum", "boostMode": "multiply",
"highlightFields": [ "highlightFields": [
"name", "name",
"displayName", "displayName",

91
openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java
View File

@ -1,91 +0,0 @@
package org.openmetadata.service.search.indexes;
import static org.junit.jupiter.api.Assertions.*;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.openmetadata.service.util.FullyQualifiedName;
class SearchIndexTest {
// Test the getFQNParts logic directly without instantiating SearchIndex
private Set<String> getFQNParts(String fqn) {
var parts = FullyQualifiedName.split(fqn);
var entityName = parts[parts.length - 1];
return FullyQualifiedName.getAllParts(fqn).stream()
.filter(part -> !part.equals(entityName))
.collect(Collectors.toSet());
}
@Test
void testGetFQNParts_excludesEntityName() {
String tableFqn = "service.database.schema.table";
Set<String> parts = getFQNParts(tableFqn);
assertFalse(parts.contains("table"), "Entity name 'table' should not be included in FQN parts");
assertTrue(parts.contains("service"));
assertTrue(parts.contains("database"));
assertTrue(parts.contains("schema"));
assertTrue(parts.contains("service.database"));
assertTrue(parts.contains("service.database.schema"));
assertTrue(parts.contains("service.database.schema.table"));
assertTrue(parts.contains("database.schema"));
assertTrue(parts.contains("schema.table"));
assertTrue(parts.contains("database.schema.table"));
assertEquals(9, parts.size());
}
@Test
void testGetFQNParts_withDifferentPatterns() {
// Test pipeline pattern: service.pipeline
String pipelineFqn = "airflow.my_pipeline";
Set<String> pipelineParts = getFQNParts(pipelineFqn);
assertFalse(pipelineParts.contains("my_pipeline"), "Entity name should not be included");
assertTrue(pipelineParts.contains("airflow"));
assertEquals(2, pipelineParts.size());
// Test dashboard pattern: service.dashboard
String dashboardFqn = "looker.sales_dashboard";
Set<String> dashboardParts = getFQNParts(dashboardFqn);
assertFalse(dashboardParts.contains("sales_dashboard"), "Entity name should not be included");
assertTrue(dashboardParts.contains("looker"));
assertEquals(2, dashboardParts.size());
// Test dashboard chart pattern: service.dashboard.chart
String chartFqn = "tableau.analytics.revenue_chart";
Set<String> chartParts = getFQNParts(chartFqn);
assertFalse(chartParts.contains("revenue_chart"), "Entity name should not be included");
assertTrue(chartParts.contains("tableau"));
assertTrue(chartParts.contains("analytics"));
assertTrue(chartParts.contains("tableau.analytics"));
assertEquals(5, chartParts.size());
}
@Test
void testGetFQNParts_withQuotedNames() {
// Test with quoted names that contain dots
String quotedFqn = "\"service.v1\".database.\"schema.prod\".\"table.users\"";
Set<String> parts = getFQNParts(quotedFqn);
// Verify that the entity name is not included
assertFalse(parts.contains("\"table.users\""), "Entity name should not be included");
assertFalse(parts.contains("table.users"), "Entity name should not be included");
// Verify other parts are included
assertTrue(parts.contains("\"service.v1\""));
assertTrue(parts.contains("database"));
assertTrue(parts.contains("\"schema.prod\""));
}
@Test
void testGetFQNParts_withSinglePart() {
// Test with a single part FQN (edge case)
String singlePartFqn = "standalone_entity";
Set<String> parts = getFQNParts(singlePartFqn);
// Should return empty set since we exclude the entity name
assertTrue(parts.isEmpty(), "Single part FQN should return empty set");
}
}

79
openmetadata-service/src/test/java/org/openmetadata/service/util/FullyQualifiedNameTest.java
View File

@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List; import java.util.List;
import java.util.Set;
import org.antlr.v4.runtime.misc.ParseCancellationException; import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -97,82 +96,4 @@ class FullyQualifiedNameTest {
assertFalse(FullyQualifiedName.isParent("a.b.c", "a.b.c")); assertFalse(FullyQualifiedName.isParent("a.b.c", "a.b.c"));
assertFalse(FullyQualifiedName.isParent("a.b c", "a.b")); assertFalse(FullyQualifiedName.isParent("a.b c", "a.b"));
} }
@Test
void test_getAllParts() {
Set<String> parts = FullyQualifiedName.getAllParts("a.b.c.d");
assertTrue(parts.contains("a"));
assertTrue(parts.contains("b"));
assertTrue(parts.contains("c"));
assertTrue(parts.contains("d"));
// Should contain top-down hierarchy
assertTrue(parts.contains("a"));
assertTrue(parts.contains("a.b"));
assertTrue(parts.contains("a.b.c"));
assertTrue(parts.contains("a.b.c.d"));
// Should contain bottom-up combinations
assertTrue(parts.contains("b.c.d"));
assertTrue(parts.contains("c.d"));
assertEquals(10, parts.size()); // 4 individual + 4 top-down + 2 bottom-up
// Test with quoted names
Set<String> quotedParts = FullyQualifiedName.getAllParts("\"a.1\".\"b.2\".c.d");
assertTrue(quotedParts.contains("\"a.1\""));
assertTrue(quotedParts.contains("\"b.2\""));
assertTrue(quotedParts.contains("c"));
assertTrue(quotedParts.contains("d"));
assertTrue(quotedParts.contains("\"a.1\".\"b.2\".c.d"));
assertTrue(quotedParts.contains("\"b.2\".c.d"));
// Test with single part
Set<String> singlePart = FullyQualifiedName.getAllParts("service");
assertEquals(1, singlePart.size());
assertTrue(singlePart.contains("service"));
}
@Test
void test_getHierarchicalParts() {
List<String> hierarchy = FullyQualifiedName.getHierarchicalParts("a.b.c.d");
assertEquals(4, hierarchy.size());
assertEquals("a", hierarchy.get(0));
assertEquals("a.b", hierarchy.get(1));
assertEquals("a.b.c", hierarchy.get(2));
assertEquals("a.b.c.d", hierarchy.get(3));
// Test with quoted names
List<String> quotedHierarchy = FullyQualifiedName.getHierarchicalParts("\"a.1\".b.\"c.3\"");
assertEquals(3, quotedHierarchy.size());
assertEquals("\"a.1\"", quotedHierarchy.get(0));
assertEquals("\"a.1\".b", quotedHierarchy.get(1));
assertEquals("\"a.1\".b.\"c.3\"", quotedHierarchy.get(2));
// Test with single part
List<String> singleHierarchy = FullyQualifiedName.getHierarchicalParts("service");
assertEquals(1, singleHierarchy.size());
assertEquals("service", singleHierarchy.getFirst());
}
@Test
void test_getAncestors() {
List<String> ancestors = FullyQualifiedName.getAncestors("a.b.c.d");
assertEquals(3, ancestors.size());
assertEquals("a.b.c", ancestors.get(0));
assertEquals("a.b", ancestors.get(1));
assertEquals("a", ancestors.get(2));
List<String> twoPartAncestors = FullyQualifiedName.getAncestors("a.b");
assertEquals(1, twoPartAncestors.size());
assertEquals("a", twoPartAncestors.getFirst());
// Test with single part (no ancestors)
List<String> noAncestors = FullyQualifiedName.getAncestors("service");
assertEquals(0, noAncestors.size());
// Test with quoted names
List<String> quotedAncestors = FullyQualifiedName.getAncestors("\"a.1\".b.\"c.3\".d");
assertEquals(3, quotedAncestors.size());
assertEquals("\"a.1\".b.\"c.3\"", quotedAncestors.get(0));
assertEquals("\"a.1\".b", quotedAncestors.get(1));
assertEquals("\"a.1\"", quotedAncestors.get(2));
}
} }

12
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts
View File

@ -1,15 +1,3 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/** /**
* This schema defines the `Database Service` is a service such as MySQL, BigQuery, * This schema defines the `Database Service` is a service such as MySQL, BigQuery,
* Redshift, Postgres, or Snowflake. Alternative terms such as Database Cluster, Database * Redshift, Postgres, or Snowflake. Alternative terms such as Database Cluster, Database

12
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts
View File

@ -1,15 +1,3 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/** /**
* Ingestion Pipeline Config is used to set up a DAG and deploy. This entity is used to * Ingestion Pipeline Config is used to set up a DAG and deploy. This entity is used to
* setup metadata/quality pipelines on Apache Airflow. * setup metadata/quality pipelines on Apache Airflow.

12
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
View File

@ -1,15 +1,3 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/** /**
* This schema defines the Pipeline Service entity, such as Airflow and Prefect. * This schema defines the Pipeline Service entity, such as Airflow and Prefect.
*/ */

12
openmetadata-ui/src/main/resources/ui/src/generated/jobs/backgroundJob.ts
View File

@ -1,15 +1,3 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/** /**
* Defines a background job that is triggered on insertion of new record in background_jobs * Defines a background job that is triggered on insertion of new record in background_jobs
* table. * table.