From 8adda4955c40dd76f66b3dac579d42db90a96e4e Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Wed, 18 Jun 2025 16:43:34 -0700 Subject: [PATCH] Revert "Issues in Search Relevancy (#21841)" (#21853) This reverts commit f388e570c1dac5b9eee31364870fb66e42715f18. --- .../examples/sample_data/datasets/tables.json | 212 ------------------ openmetadata-service/pom.xml | 6 - .../search/SearchSourceBuilderFactory.java | 19 +- .../elasticsearch/ElasticSearchClient.java | 6 +- .../ElasticSearchSourceBuilderFactory.java | 8 +- .../service/search/indexes/SearchIndex.java | 29 ++- .../search/opensearch/OpenSearchClient.java | 15 +- .../OpenSearchSourceBuilderFactory.java | 8 +- .../service/util/FullyQualifiedName.java | 61 ----- .../data/searchSettings/searchSettings.json | 34 +-- .../search/indexes/SearchIndexTest.java | 91 -------- .../service/util/FullyQualifiedNameTest.java | 79 ------- .../entity/services/databaseService.ts | 12 - .../ingestionPipelines/ingestionPipeline.ts | 12 - .../entity/services/pipelineService.ts | 12 - .../ui/src/generated/jobs/backgroundJob.ts | 12 - 16 files changed, 45 insertions(+), 571 deletions(-) delete mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java diff --git a/ingestion/examples/sample_data/datasets/tables.json b/ingestion/examples/sample_data/datasets/tables.json index d1598d02832..86402a83b9f 100644 --- a/ingestion/examples/sample_data/datasets/tables.json +++ b/ingestion/examples/sample_data/datasets/tables.json @@ -32491,218 +32491,6 @@ }, "sourceUrl": "http://localhost:8080/test_long_description_table", "tags": [] - }, - { - "name": "ssot_utilization_detail", - "description": "A detail of each agent's time throughout the day. Individual events in each system are collapsed by aux_code, aux_channel, worktype_name, and reporting_date.", - "version": 0.5, - "updatedAt": 1638354087391, - "updatedBy": "anonymous", - "tableType": "Regular", - "columns": [ - { - "name": "reporting_date", - "dataType": "DATE", - "dataTypeDisplay": "date", - "description": "The date the event began.", - "tags": [], - "ordinalPosition": 1 - }, - { - "name": "agent_id", - "dataType": "VARCHAR", - "dataLength": 100, - "dataTypeDisplay": "varchar", - "description": "The unique identifier for the agent from customer_success.remitly_employee.", - "tags": [], - "ordinalPosition": 2 - }, - { - "name": "agent_email", - "dataType": "VARCHAR", - "dataLength": 100, - "dataTypeDisplay": "varchar", - "description": "The agent email.", - "tags": [], - "ordinalPosition": 3 - } - ], - "database": { - "id": "50da1ff8-4e1d-4967-8931-45edbf4dd908", - "type": "database", - "name": "sample_data.ecommerce_db", - "description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.", - "href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908" - }, - "tags": [], - "usageSummary": { - "dailyStats": { - "count": 0, - "percentileRank": 0 - }, - "weeklyStats": { - "count": 0, - "percentileRank": 0 - }, - "monthlyStats": { - "count": 0, - "percentileRank": 0 - }, - "date": "2021-12-01" - }, - "followers": [] - }, - { - "name": "itm_utilization_detail_login_events", - "description": "An intermediate table used to create ssot_utilization_detail. This table cleans up the \"CS Platform: login\" event dimensions to be congruent with downstream analytics.", - "version": 0.3, - "updatedAt": 1638354087391, - "updatedBy": "anonymous", - "tableType": "Regular", - "columns": [ - { - "name": "reporting_date", - "dataType": "DATE", - "dataTypeDisplay": "date", - "description": "Event date", - "tags": [], - "ordinalPosition": 1 - }, - { - "name": "agent_id", - "dataType": "VARCHAR", - "dataLength": 100, - "dataTypeDisplay": "varchar", - "description": "Agent identifier", - "tags": [], - "ordinalPosition": 2 - } - ], - "database": { - "id": "50da1ff8-4e1d-4967-8931-45edbf4dd908", - "type": "database", - "name": "sample_data.ecommerce_db", - "description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.", - "href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908" - }, - "tags": [], - "usageSummary": { - "dailyStats": { - "count": 10, - "percentileRank": 30 - }, - "weeklyStats": { - "count": 50, - "percentileRank": 40 - }, - "monthlyStats": { - "count": 200, - "percentileRank": 50 - }, - "date": "2021-12-01" - }, - "followers": [] - }, - { - "name": "customer_metrics_daily", - "description": "Daily customer metrics aggregated from various sources including ssot_utilization_detail and other operational tables.", - "version": 0.2, - "updatedAt": 1638354087391, - "updatedBy": "anonymous", - "tableType": "Regular", - "columns": [ - { - "name": "metric_date", - "dataType": "DATE", - "dataTypeDisplay": "date", - "description": "Date of the metric", - "tags": [], - "ordinalPosition": 1 - }, - { - "name": "customer_count", - "dataType": "INT", - "dataTypeDisplay": "integer", - "description": "Total number of customers", - "tags": [], - "ordinalPosition": 2 - } - ], - "database": { - "id": "50da1ff8-4e1d-4967-8931-45edbf4dd908", - "type": "database", - "name": "sample_data.ecommerce_db", - "description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.", - "href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908" - }, - "tags": [], - "usageSummary": { - "dailyStats": { - "count": 100, - "percentileRank": 70 - }, - "weeklyStats": { - "count": 500, - "percentileRank": 80 - }, - "monthlyStats": { - "count": 2000, - "percentileRank": 90 - }, - "date": "2021-12-01" - }, - "followers": [] - }, - { - "name": "agent_performance_summary", - "description": "Summary of agent performance metrics derived from multiple tables including ssot_utilization_detail for comprehensive reporting.", - "version": 0.4, - "updatedAt": 1638354087391, - "updatedBy": "anonymous", - "tableType": "Regular", - "columns": [ - { - "name": "agent_id", - "dataType": "VARCHAR", - "dataLength": 100, - "dataTypeDisplay": "varchar", - "description": "Agent identifier", - "tags": [], - "ordinalPosition": 1 - }, - { - "name": "performance_score", - "dataType": "DECIMAL", - "dataTypeDisplay": "decimal", - "description": "Overall performance score", - "tags": [], - "ordinalPosition": 2 - } - ], - "database": { - "id": "50da1ff8-4e1d-4967-8931-45edbf4dd908", - "type": "database", - "name": "sample_data.ecommerce_db", - "description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.", - "href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908" - }, - "tags": [], - "usageSummary": { - "dailyStats": { - "count": 50, - "percentileRank": 60 - }, - "weeklyStats": { - "count": 250, - "percentileRank": 65 - }, - "monthlyStats": { - "count": 1000, - "percentileRank": 70 - }, - "date": "2021-12-01" - }, - "followers": [] } ] } \ No newline at end of file diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml index cde26c71c58..3a676b70b16 100644 --- a/openmetadata-service/pom.xml +++ b/openmetadata-service/pom.xml @@ -948,12 +948,6 @@ opensearch-rest-high-level-client test - - org.open-metadata - opensearch-deps - 1.8.0-SNAPSHOT - compile - diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchSourceBuilderFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchSourceBuilderFactory.java index 4143038fec3..1e79c046c93 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchSourceBuilderFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchSourceBuilderFactory.java @@ -72,20 +72,6 @@ public interface SearchSourceBuilderFactory { * @return a search source builder configured for the specific entity type */ default S getSearchSourceBuilder(String index, String q, int from, int size) { - return getSearchSourceBuilder(index, q, from, size, false); - } - - /** - * Get the appropriate search source builder based on the index name. - * - * @param index the index name - * @param q the search query - * @param from the starting offset - * @param size the number of results to return - * @param explain whether to include explanation of the search results - * @return a search source builder configured for the specific entity type - */ - default S getSearchSourceBuilder(String index, String q, int from, int size, boolean explain) { String indexName = Entity.getSearchRepository().getIndexNameWithoutAlias(index); if (isTimeSeriesIndex(indexName)) { @@ -101,7 +87,7 @@ public interface SearchSourceBuilderFactory { } if (isDataAssetIndex(indexName)) { - return buildDataAssetSearchBuilder(indexName, q, from, size, explain); + return buildDataAssetSearchBuilder(indexName, q, from, size); } if (indexName.equals("all") || indexName.equals("dataAsset")) { @@ -119,9 +105,6 @@ public interface SearchSourceBuilderFactory { S buildDataAssetSearchBuilder(String indexName, String query, int from, int size); - S buildDataAssetSearchBuilder( - String indexName, String query, int from, int size, boolean explain); - S buildCommonSearchBuilder(String query, int from, int size); S buildUserOrTeamSearchBuilder(String query, int from, int size); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 130456f19b6..965f3099111 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -366,11 +366,7 @@ public class ElasticSearchClient implements SearchClient { new ElasticSearchSourceBuilderFactory(searchSettings); SearchSourceBuilder searchSourceBuilder = searchBuilderFactory.getSearchSourceBuilder( - request.getIndex(), - request.getQuery(), - request.getFrom(), - request.getSize(), - request.getExplain()); + request.getIndex(), request.getQuery(), request.getFrom(), request.getSize()); buildSearchRBACQuery(subjectContext, searchSourceBuilder); // Add Filter diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java index b8c6f5a0542..b10c99689ad 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java @@ -163,12 +163,6 @@ public class ElasticSearchSourceBuilderFactory @Override public SearchSourceBuilder buildDataAssetSearchBuilder( String indexName, String query, int from, int size) { - return buildDataAssetSearchBuilder(indexName, query, from, size, false); - } - - @Override - public SearchSourceBuilder buildDataAssetSearchBuilder( - String indexName, String query, int from, int size, boolean explain) { AssetTypeConfiguration assetConfig = findAssetTypeConfig(indexName, searchSettings); Map fuzzyFields; Map nonFuzzyFields; @@ -318,7 +312,7 @@ public class ElasticSearchSourceBuilderFactory } addConfiguredAggregations(searchSourceBuilder, assetConfig); - searchSourceBuilder.explain(explain); + searchSourceBuilder.explain(true); return searchSourceBuilder; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index 87374d8101d..bad99694900 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -19,11 +19,11 @@ import static org.openmetadata.service.util.FullyQualifiedName.getParentFQN; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.openmetadata.schema.EntityInterface; @@ -128,12 +128,29 @@ public interface SearchIndex { } default Set getFQNParts(String fqn) { - var parts = FullyQualifiedName.split(fqn); - var entityName = parts[parts.length - 1]; + Set fqnParts = new HashSet<>(); + fqnParts.add(fqn); + String parent = FullyQualifiedName.getParentFQN(fqn); + while (parent != null) { + fqnParts.add(parent); + parent = FullyQualifiedName.getParentFQN(parent); + } + return fqnParts; + } - return FullyQualifiedName.getAllParts(fqn).stream() - .filter(part -> !part.equals(entityName)) - .collect(Collectors.toSet()); + // Add suggest inputs to fqnParts to support partial/wildcard search on names. + // In some case of basic Test suite name is not part of the fullyQualifiedName, so it must be + // added separately. + default Set getFQNParts(String fqn, List fqnSplits) { + Set fqnParts = new HashSet<>(); + fqnParts.add(fqn); + String parent = FullyQualifiedName.getParentFQN(fqn); + while (parent != null) { + fqnParts.add(parent); + parent = FullyQualifiedName.getParentFQN(parent); + } + fqnParts.addAll(fqnSplits); + return fqnParts; } default List getEntitiesWithDisplayName(List entities) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 7a21b82048c..26e1abc8dd5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -137,7 +137,6 @@ import os.org.opensearch.action.delete.DeleteRequest; import os.org.opensearch.action.get.GetRequest; import os.org.opensearch.action.get.GetResponse; import os.org.opensearch.action.search.SearchResponse; -import os.org.opensearch.action.search.SearchType; import os.org.opensearch.action.support.WriteRequest; import os.org.opensearch.action.support.master.AcknowledgedResponse; import os.org.opensearch.action.update.UpdateRequest; @@ -385,11 +384,7 @@ public class OpenSearchClient implements SearchClient { new OpenSearchSourceBuilderFactory(searchSettings); SearchSourceBuilder searchSourceBuilder = searchBuilderFactory.getSearchSourceBuilder( - request.getIndex(), - request.getQuery(), - request.getFrom(), - request.getSize(), - request.getExplain()); + request.getIndex(), request.getQuery(), request.getFrom(), request.getSize()); buildSearchRBACQuery(subjectContext, searchSourceBuilder); @@ -514,10 +509,6 @@ public class OpenSearchClient implements SearchClient { os.org.opensearch.action.search.SearchRequest searchRequest = new os.org.opensearch.action.search.SearchRequest(request.getIndex()); searchRequest.source(searchSourceBuilder); - - // Use DFS Query Then Fetch for consistent scoring across shards - searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH); - os.org.opensearch.action.search.SearchResponse response = client.search(searchRequest, os.org.opensearch.client.RequestOptions.DEFAULT); if (response.getHits() != null @@ -550,10 +541,6 @@ public class OpenSearchClient implements SearchClient { os.org.opensearch.action.search.SearchRequest osRequest = new os.org.opensearch.action.search.SearchRequest(request.getIndex()); osRequest.source(searchSourceBuilder); - - // Use DFS Query Then Fetch for consistent scoring across shards - osRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH); - getSearchBuilderFactory().addAggregationsToNLQQuery(searchSourceBuilder, request.getIndex()); SearchResponse searchResponse = client.search(osRequest, OPENSEARCH_REQUEST_OPTIONS); return Response.status(Response.Status.OK).entity(searchResponse.toString()).build(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java index 8e3d55d9ce3..259121b2269 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchSourceBuilderFactory.java @@ -162,12 +162,6 @@ public class OpenSearchSourceBuilderFactory @Override public SearchSourceBuilder buildDataAssetSearchBuilder( String indexName, String query, int from, int size) { - return buildDataAssetSearchBuilder(indexName, query, from, size, false); - } - - @Override - public SearchSourceBuilder buildDataAssetSearchBuilder( - String indexName, String query, int from, int size, boolean explain) { AssetTypeConfiguration assetConfig = findAssetTypeConfig(indexName, searchSettings); Map fuzzyFields; Map nonFuzzyFields; @@ -318,7 +312,7 @@ public class OpenSearchSourceBuilderFactory } addConfiguredAggregations(searchSourceBuilder, assetConfig); - searchSourceBuilder.explain(explain); + searchSourceBuilder.explain(true); return searchSourceBuilder; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/FullyQualifiedName.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/FullyQualifiedName.java index 98689127ef9..4fcd54102e2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/FullyQualifiedName.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/FullyQualifiedName.java @@ -14,9 +14,7 @@ package org.openmetadata.service.util; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.antlr.v4.runtime.BailErrorStrategy; @@ -208,63 +206,4 @@ public class FullyQualifiedName { public static String getColumnName(String columnFQN) { return FullyQualifiedName.split(columnFQN)[4]; // Get from column name from FQN } - - /** - * Generates all possible FQN parts for search and matching purposes. - * For example, given FQN "service.database.schema.table", this method generates: - * - Full hierarchy: "service", "service.database", "service.database.schema", "service.database.schema.table" - * - Individual parts: "service", "database", "schema", "table" - * - Bottom-up combinations: "database.schema.table", "schema.table", "table" - * - * @param fqn The fully qualified name to generate parts from - * @return Set of all possible FQN parts - */ - public static Set getAllParts(String fqn) { - var parts = split(fqn); - var fqnParts = new HashSet(); - - // Generate all possible sub-paths - for (int start = 0; start < parts.length; start++) { - for (int end = start + 1; end <= parts.length; end++) { - var subPath = - String.join(Entity.SEPARATOR, java.util.Arrays.copyOfRange(parts, start, end)); - fqnParts.add(subPath); - } - } - - return fqnParts; - } - - /** - * Generates hierarchical FQN parts from root to the full FQN. - * For example, given FQN "service.database.schema.table", this method generates: - * ["service", "service.database", "service.database.schema", "service.database.schema.table"] - * - * @param fqn The fully qualified name to generate hierarchy from - * @return List of hierarchical FQN parts from root to full FQN - */ - public static List getHierarchicalParts(String fqn) { - var parts = split(fqn); - return java.util.stream.IntStream.rangeClosed(1, parts.length) - .mapToObj(i -> String.join(Entity.SEPARATOR, java.util.Arrays.copyOfRange(parts, 0, i))) - .toList(); - } - - /** - * Gets all ancestor FQNs for a given FQN. - * For example, given FQN "service.database.schema.table", this method returns: - * ["service.database.schema", "service.database", "service"] - * - * @param fqn The fully qualified name to get ancestors from - * @return List of ancestor FQNs (excluding the input FQN itself) - */ - public static List getAncestors(String fqn) { - var parts = split(fqn); - return java.util.stream.IntStream.range(1, parts.length) - .mapToObj( - i -> - String.join( - Entity.SEPARATOR, java.util.Arrays.copyOfRange(parts, 0, parts.length - i))) - .toList(); - } } diff --git a/openmetadata-service/src/main/resources/json/data/searchSettings/searchSettings.json b/openmetadata-service/src/main/resources/json/data/searchSettings/searchSettings.json index 4588ad957e2..984e629b27a 100644 --- a/openmetadata-service/src/main/resources/json/data/searchSettings/searchSettings.json +++ b/openmetadata-service/src/main/resources/json/data/searchSettings/searchSettings.json @@ -154,7 +154,7 @@ "aggregations": [ ], "scoreMode": "sum", - "boostMode": "sum" + "boostMode": "multiply" }, { "assetType": "databaseSchema", @@ -200,7 +200,7 @@ "aggregations": [ ], "scoreMode": "sum", - "boostMode": "sum" + "boostMode": "multiply" }, { "assetType": "table", @@ -271,7 +271,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "description", @@ -341,7 +341,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "description", @@ -400,7 +400,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "description", @@ -468,7 +468,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -571,7 +571,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -641,7 +641,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -701,7 +701,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -762,7 +762,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -823,7 +823,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -888,7 +888,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -976,7 +976,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -1027,7 +1027,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -1073,7 +1073,7 @@ "aggregations": [ ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -1127,7 +1127,7 @@ "aggregations": [ ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", @@ -1204,7 +1204,7 @@ } ], "scoreMode": "sum", - "boostMode": "sum", + "boostMode": "multiply", "highlightFields": [ "name", "displayName", diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java deleted file mode 100644 index 8a4cf43c6ce..00000000000 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.openmetadata.service.search.indexes; - -import static org.junit.jupiter.api.Assertions.*; - -import java.util.Set; -import java.util.stream.Collectors; -import org.junit.jupiter.api.Test; -import org.openmetadata.service.util.FullyQualifiedName; - -class SearchIndexTest { - - // Test the getFQNParts logic directly without instantiating SearchIndex - private Set getFQNParts(String fqn) { - var parts = FullyQualifiedName.split(fqn); - var entityName = parts[parts.length - 1]; - - return FullyQualifiedName.getAllParts(fqn).stream() - .filter(part -> !part.equals(entityName)) - .collect(Collectors.toSet()); - } - - @Test - void testGetFQNParts_excludesEntityName() { - String tableFqn = "service.database.schema.table"; - Set parts = getFQNParts(tableFqn); - assertFalse(parts.contains("table"), "Entity name 'table' should not be included in FQN parts"); - - assertTrue(parts.contains("service")); - assertTrue(parts.contains("database")); - assertTrue(parts.contains("schema")); - assertTrue(parts.contains("service.database")); - assertTrue(parts.contains("service.database.schema")); - assertTrue(parts.contains("service.database.schema.table")); - assertTrue(parts.contains("database.schema")); - assertTrue(parts.contains("schema.table")); - assertTrue(parts.contains("database.schema.table")); - assertEquals(9, parts.size()); - } - - @Test - void testGetFQNParts_withDifferentPatterns() { - // Test pipeline pattern: service.pipeline - String pipelineFqn = "airflow.my_pipeline"; - Set pipelineParts = getFQNParts(pipelineFqn); - assertFalse(pipelineParts.contains("my_pipeline"), "Entity name should not be included"); - assertTrue(pipelineParts.contains("airflow")); - assertEquals(2, pipelineParts.size()); - - // Test dashboard pattern: service.dashboard - String dashboardFqn = "looker.sales_dashboard"; - Set dashboardParts = getFQNParts(dashboardFqn); - assertFalse(dashboardParts.contains("sales_dashboard"), "Entity name should not be included"); - assertTrue(dashboardParts.contains("looker")); - assertEquals(2, dashboardParts.size()); - - // Test dashboard chart pattern: service.dashboard.chart - String chartFqn = "tableau.analytics.revenue_chart"; - Set chartParts = getFQNParts(chartFqn); - assertFalse(chartParts.contains("revenue_chart"), "Entity name should not be included"); - assertTrue(chartParts.contains("tableau")); - assertTrue(chartParts.contains("analytics")); - assertTrue(chartParts.contains("tableau.analytics")); - assertEquals(5, chartParts.size()); - } - - @Test - void testGetFQNParts_withQuotedNames() { - // Test with quoted names that contain dots - String quotedFqn = "\"service.v1\".database.\"schema.prod\".\"table.users\""; - Set parts = getFQNParts(quotedFqn); - - // Verify that the entity name is not included - assertFalse(parts.contains("\"table.users\""), "Entity name should not be included"); - assertFalse(parts.contains("table.users"), "Entity name should not be included"); - - // Verify other parts are included - assertTrue(parts.contains("\"service.v1\"")); - assertTrue(parts.contains("database")); - assertTrue(parts.contains("\"schema.prod\"")); - } - - @Test - void testGetFQNParts_withSinglePart() { - // Test with a single part FQN (edge case) - String singlePartFqn = "standalone_entity"; - Set parts = getFQNParts(singlePartFqn); - - // Should return empty set since we exclude the entity name - assertTrue(parts.isEmpty(), "Single part FQN should return empty set"); - } -} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/FullyQualifiedNameTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/FullyQualifiedNameTest.java index b802b88e21a..142d6a30ffd 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/FullyQualifiedNameTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/FullyQualifiedNameTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import java.util.Set; import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; @@ -97,82 +96,4 @@ class FullyQualifiedNameTest { assertFalse(FullyQualifiedName.isParent("a.b.c", "a.b.c")); assertFalse(FullyQualifiedName.isParent("a.b c", "a.b")); } - - @Test - void test_getAllParts() { - Set parts = FullyQualifiedName.getAllParts("a.b.c.d"); - assertTrue(parts.contains("a")); - assertTrue(parts.contains("b")); - assertTrue(parts.contains("c")); - assertTrue(parts.contains("d")); - // Should contain top-down hierarchy - assertTrue(parts.contains("a")); - assertTrue(parts.contains("a.b")); - assertTrue(parts.contains("a.b.c")); - assertTrue(parts.contains("a.b.c.d")); - // Should contain bottom-up combinations - assertTrue(parts.contains("b.c.d")); - assertTrue(parts.contains("c.d")); - assertEquals(10, parts.size()); // 4 individual + 4 top-down + 2 bottom-up - - // Test with quoted names - Set quotedParts = FullyQualifiedName.getAllParts("\"a.1\".\"b.2\".c.d"); - assertTrue(quotedParts.contains("\"a.1\"")); - assertTrue(quotedParts.contains("\"b.2\"")); - assertTrue(quotedParts.contains("c")); - assertTrue(quotedParts.contains("d")); - assertTrue(quotedParts.contains("\"a.1\".\"b.2\".c.d")); - assertTrue(quotedParts.contains("\"b.2\".c.d")); - - // Test with single part - Set singlePart = FullyQualifiedName.getAllParts("service"); - assertEquals(1, singlePart.size()); - assertTrue(singlePart.contains("service")); - } - - @Test - void test_getHierarchicalParts() { - List hierarchy = FullyQualifiedName.getHierarchicalParts("a.b.c.d"); - assertEquals(4, hierarchy.size()); - assertEquals("a", hierarchy.get(0)); - assertEquals("a.b", hierarchy.get(1)); - assertEquals("a.b.c", hierarchy.get(2)); - assertEquals("a.b.c.d", hierarchy.get(3)); - - // Test with quoted names - List quotedHierarchy = FullyQualifiedName.getHierarchicalParts("\"a.1\".b.\"c.3\""); - assertEquals(3, quotedHierarchy.size()); - assertEquals("\"a.1\"", quotedHierarchy.get(0)); - assertEquals("\"a.1\".b", quotedHierarchy.get(1)); - assertEquals("\"a.1\".b.\"c.3\"", quotedHierarchy.get(2)); - - // Test with single part - List singleHierarchy = FullyQualifiedName.getHierarchicalParts("service"); - assertEquals(1, singleHierarchy.size()); - assertEquals("service", singleHierarchy.getFirst()); - } - - @Test - void test_getAncestors() { - List ancestors = FullyQualifiedName.getAncestors("a.b.c.d"); - assertEquals(3, ancestors.size()); - assertEquals("a.b.c", ancestors.get(0)); - assertEquals("a.b", ancestors.get(1)); - assertEquals("a", ancestors.get(2)); - - List twoPartAncestors = FullyQualifiedName.getAncestors("a.b"); - assertEquals(1, twoPartAncestors.size()); - assertEquals("a", twoPartAncestors.getFirst()); - - // Test with single part (no ancestors) - List noAncestors = FullyQualifiedName.getAncestors("service"); - assertEquals(0, noAncestors.size()); - - // Test with quoted names - List quotedAncestors = FullyQualifiedName.getAncestors("\"a.1\".b.\"c.3\".d"); - assertEquals(3, quotedAncestors.size()); - assertEquals("\"a.1\".b.\"c.3\"", quotedAncestors.get(0)); - assertEquals("\"a.1\".b", quotedAncestors.get(1)); - assertEquals("\"a.1\"", quotedAncestors.get(2)); - } } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts index b91253c36eb..66b51e8b6d2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts @@ -1,15 +1,3 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ /** * This schema defines the `Database Service` is a service such as MySQL, BigQuery, * Redshift, Postgres, or Snowflake. Alternative terms such as Database Cluster, Database diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts index 64db32c1127..d125de3f6ff 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts @@ -1,15 +1,3 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ /** * Ingestion Pipeline Config is used to set up a DAG and deploy. This entity is used to * setup metadata/quality pipelines on Apache Airflow. diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts index d3ef970388b..1e2120a16de 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts @@ -1,15 +1,3 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ /** * This schema defines the Pipeline Service entity, such as Airflow and Prefect. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/jobs/backgroundJob.ts b/openmetadata-ui/src/main/resources/ui/src/generated/jobs/backgroundJob.ts index 69cf560b48a..5d3f0f9e7d2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/jobs/backgroundJob.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/jobs/backgroundJob.ts @@ -1,15 +1,3 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ /** * Defines a background job that is triggered on insertion of new record in background_jobs * table.