Added source url to search indexes and missing entites (#12773)

* added sourceUrl to missing entites and added sourceUrl to search indexes * edited desc in json files * edited desc in json files * working on adding api for getting entity from sourceUrl * added API for getting Entity from sourceUrl * added api to fetch entity based on sourceUrl * remove desc * working on adding alias
2025-10-22 06:14:56 +00:00 · 2023-08-16 15:56:27 +05:30 · 2023-08-16 15:56:27 +05:30 · ad3871b3bc
commit ad3871b3bc
parent 911b41f7f3
23 changed files with 179 additions and 2 deletions
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
@ -182,6 +182,26 @@ public class SearchResource {
    return searchClient.search(request);
  }

+  @GET
+  @Path("/sourceUrl")
+  @Operation(
+      operationId = "searchEntitiesWithSourceUrl",
+      summary = "Search entities",
+      responses = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "search response",
+            content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchResponse.class)))
+      })
+  public Response searchBySourceUrl(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(description = "source url") @QueryParam("sourceUrl") String sourceUrl)
+      throws IOException {
+
+    return searchClient.searchBySourceUrl(sourceUrl);
+  }
+
  @GET
  @Path("/suggest")
  @Operation(
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
@ -54,6 +54,8 @@ public interface SearchClient {

  Response search(SearchRequest request) throws IOException;

+  Response searchBySourceUrl(String sourceUrl) throws IOException;
+
  Response aggregate(String index, String fieldName, String value, String query) throws IOException;

  Response suggest(SearchRequest request) throws IOException;
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java
@ -2,7 +2,10 @@ package org.openmetadata.service.search.elasticSearch;

 import static javax.ws.rs.core.Response.Status.OK;
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
-import static org.openmetadata.schema.type.EventType.*;
+import static org.openmetadata.schema.type.EventType.ENTITY_DELETED;
+import static org.openmetadata.schema.type.EventType.ENTITY_RESTORED;
+import static org.openmetadata.schema.type.EventType.ENTITY_SOFT_DELETED;
+import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED;
 import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
 import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
 import static org.openmetadata.service.Entity.FIELD_NAME;
@ -52,6 +55,7 @@ import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
@ -198,6 +202,14 @@ public class ElasticSearchClientImpl implements SearchClient {
        request.source(elasticSearchIndexMapping, XContentType.JSON);
        CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
        LOG.info("{} Created {}", elasticSearchIndexType.indexName, createIndexResponse.isAcknowledged());
+        // creating alias for indexes
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+        IndicesAliasesRequest.AliasActions aliasAction =
+            IndicesAliasesRequest.AliasActions.add()
+                .index(elasticSearchIndexType.indexName)
+                .alias("sourceUrlSearchAlias");
+        aliasesRequest.addAliasAction(aliasAction);
+        client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
      }
      elasticSearchIndexes.put(elasticSearchIndexType, IndexUtil.ElasticSearchIndexStatus.CREATED);
    } catch (Exception e) {
@ -219,6 +231,14 @@ public class ElasticSearchClientImpl implements SearchClient {
      String elasticSearchIndexMapping = getIndexMapping(elasticSearchIndexType, lang);
      ENTITY_TO_MAPPING_SCHEMA_MAP.put(
          elasticSearchIndexType.entityType, JsonUtils.getMap(JsonUtils.readJson(elasticSearchIndexMapping)));
+      // creating alias for indexes
+      IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+      IndicesAliasesRequest.AliasActions aliasAction =
+          IndicesAliasesRequest.AliasActions.add()
+              .index(elasticSearchIndexType.indexName)
+              .alias("sourceUrlSearchAlias");
+      aliasesRequest.addAliasAction(aliasAction);
+      client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
      if (exists) {
        PutMappingRequest request = new PutMappingRequest(elasticSearchIndexType.indexName);
        request.source(elasticSearchIndexMapping, XContentType.JSON);
@ -247,6 +267,14 @@ public class ElasticSearchClientImpl implements SearchClient {
      gRequest.local(false);
      boolean exists = client.indices().exists(gRequest, RequestOptions.DEFAULT);
      if (exists) {
+        // deleting alias for indexes
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+        IndicesAliasesRequest.AliasActions aliasAction =
+            IndicesAliasesRequest.AliasActions.remove()
+                .index(elasticSearchIndexType.indexName)
+                .alias("sourceUrlSearchAlias");
+        aliasesRequest.addAliasAction(aliasAction);
+        client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
        DeleteIndexRequest request = new DeleteIndexRequest(elasticSearchIndexType.indexName);
        AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
        LOG.info("{} Deleted {}", elasticSearchIndexType.indexName, deleteIndexResponse.isAcknowledged());
@ -363,6 +391,22 @@ public class ElasticSearchClientImpl implements SearchClient {
    return Response.status(OK).entity(response).build();
  }

+  /**
+   * @param sourceUrl
+   * @return
+   */
+  @Override
+  public Response searchBySourceUrl(String sourceUrl) throws IOException {
+    QueryBuilder wildcardQuery = QueryBuilders.queryStringQuery(sourceUrl).field("sourceUrl").escape(true);
+    org.elasticsearch.action.search.SearchRequest searchRequest =
+        new org.elasticsearch.action.search.SearchRequest("sourceUrlSearchAlias");
+    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+    searchSourceBuilder.query(wildcardQuery);
+    searchRequest.source(searchSourceBuilder);
+    String response = client.search(searchRequest, RequestOptions.DEFAULT).toString();
+    return Response.status(OK).entity(response).build();
+  }
+
  @Override
  public Response aggregate(String index, String fieldName, String value, String query) throws IOException {
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java
@ -2,7 +2,10 @@ package org.openmetadata.service.search.openSearch;

 import static javax.ws.rs.core.Response.Status.OK;
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
-import static org.openmetadata.schema.type.EventType.*;
+import static org.openmetadata.schema.type.EventType.ENTITY_DELETED;
+import static org.openmetadata.schema.type.EventType.ENTITY_RESTORED;
+import static org.openmetadata.schema.type.EventType.ENTITY_SOFT_DELETED;
+import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED;
 import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
 import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
 import static org.openmetadata.service.Entity.FIELD_NAME;
@ -92,6 +95,7 @@ import org.openmetadata.service.search.indexes.TestCaseIndex;
 import org.openmetadata.service.search.indexes.UserIndex;
 import org.openmetadata.service.util.JsonUtils;
 import org.opensearch.OpenSearchException;
+import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.bulk.BulkItemResponse;
 import org.opensearch.action.bulk.BulkRequest;
@ -189,6 +193,14 @@ public class OpenSearchClientImpl implements SearchClient {
        request.source(elasticSearchIndexMapping, XContentType.JSON);
        CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
        LOG.info("{} Created {}", elasticSearchIndexType.indexName, createIndexResponse.isAcknowledged());
+        // creating alias for indexes
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+        IndicesAliasesRequest.AliasActions aliasAction =
+            IndicesAliasesRequest.AliasActions.add()
+                .index(elasticSearchIndexType.indexName)
+                .alias("sourceUrlSearchAlias");
+        aliasesRequest.addAliasAction(aliasAction);
+        client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
      }
      elasticSearchIndexes.put(elasticSearchIndexType, IndexUtil.ElasticSearchIndexStatus.CREATED);
    } catch (Exception e) {
@ -211,6 +223,14 @@ public class OpenSearchClientImpl implements SearchClient {
      String elasticSearchIndexMapping = getIndexMapping(elasticSearchIndexType, lang);
      ENTITY_TO_MAPPING_SCHEMA_MAP.put(
          elasticSearchIndexType.entityType, JsonUtils.getMap(JsonUtils.readJson(elasticSearchIndexMapping)));
+      // creating alias for indexes
+      IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+      IndicesAliasesRequest.AliasActions aliasAction =
+          IndicesAliasesRequest.AliasActions.add()
+              .index(elasticSearchIndexType.indexName)
+              .alias("sourceUrlSearchAlias");
+      aliasesRequest.addAliasAction(aliasAction);
+      client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
      if (exists) {
        PutMappingRequest request = new PutMappingRequest(elasticSearchIndexType.indexName);
        request.source(elasticSearchIndexMapping, XContentType.JSON);
@ -239,6 +259,14 @@ public class OpenSearchClientImpl implements SearchClient {
      gRequest.local(false);
      boolean exists = client.indices().exists(gRequest, RequestOptions.DEFAULT);
      if (exists) {
+        // deleting alias for indexes
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+        IndicesAliasesRequest.AliasActions aliasAction =
+            IndicesAliasesRequest.AliasActions.remove()
+                .index(elasticSearchIndexType.indexName)
+                .alias("sourceUrlSearchAlias");
+        aliasesRequest.addAliasAction(aliasAction);
+        client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
        DeleteIndexRequest request = new DeleteIndexRequest(elasticSearchIndexType.indexName);
        AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
        LOG.info("{} Deleted {}", elasticSearchIndexType.indexName, deleteIndexResponse.isAcknowledged());
@ -357,6 +385,22 @@ public class OpenSearchClientImpl implements SearchClient {
    return Response.status(OK).entity(response).build();
  }

+  /**
+   * @param sourceUrl
+   * @return
+   */
+  @Override
+  public Response searchBySourceUrl(String sourceUrl) throws IOException {
+    QueryBuilder wildcardQuery = QueryBuilders.queryStringQuery(sourceUrl).field("sourceUrl").escape(true);
+    org.opensearch.action.search.SearchRequest searchRequest =
+        new org.opensearch.action.search.SearchRequest("sourceUrlSearchAlias");
+    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+    searchSourceBuilder.query(wildcardQuery);
+    searchRequest.source(searchSourceBuilder);
+    String response = client.search(searchRequest, RequestOptions.DEFAULT).toString();
+    return Response.status(OK).entity(response).build();
+  }
+
  public Response aggregate(String index, String fieldName, String value, String query) throws IOException {
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    XContentParser filterParser =
--- a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json
@ -87,6 +87,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "parent": {
        "properties": {
          "id": {
--- a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json
@ -95,6 +95,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "columns": {
        "properties": {
          "name": {
--- a/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json
@ -91,6 +91,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "messageSchema": {
        "properties": {
          "schemaText": {
--- a/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json
@ -83,6 +83,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "parent": {
        "properties": {
          "id": {
--- a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json
@ -105,6 +105,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "columns": {
        "properties": {
          "name": {
--- a/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json
@ -94,6 +94,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "messageSchema": {
        "properties": {
          "schemaType": {
--- a/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json
@ -78,6 +78,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "parent": {
        "properties": {
          "id": {
--- a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json
@ -70,6 +70,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "columns": {
        "properties": {
          "name": {
--- a/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json
@ -63,6 +63,9 @@
      "href": {
        "type": "text"
      },
+      "sourceUrl": {
+        "type": "text"
+      },
      "messageSchema": {
        "properties": {
          "schemaType": {
--- a/openmetadata-spec/src/main/resources/json/schema/api/data/createContainer.json
+++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createContainer.json
@ -72,6 +72,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of container.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Fully qualified name of the domain the Container belongs to.",
      "type": "string"
--- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json
+++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json
@ -49,6 +49,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of database.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Fully qualified name of the domain the Database belongs to.",
      "type": "string"
--- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json
+++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json
@ -45,6 +45,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of database schema.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Fully qualified name of the domain the Database Schema belongs to.",
      "type": "string"
--- a/openmetadata-spec/src/main/resources/json/schema/api/data/createMlModel.json
+++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createMlModel.json
@ -76,6 +76,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of mlModel.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Fully qualified name of the domain the MLModel belongs to.",
      "type": "string"
--- a/openmetadata-spec/src/main/resources/json/schema/api/data/createTopic.json
+++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createTopic.json
@ -80,6 +80,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of topic.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Fully qualified name of the domain the Topic belongs to.",
      "type": "string",
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/container.json
@ -168,6 +168,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of container.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Domain the Container belongs to. When not set, the Container inherits the domain from the storage service it belongs to.",
      "$ref": "../../type/entityReference.json"
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/database.json
@ -108,6 +108,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of database.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Domain the Database belongs to. When not set, the Database inherits the domain from the database service it belongs to.",
      "$ref": "../../type/entityReference.json"
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json
@ -103,6 +103,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of database schema.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Domain the Database Schema belongs to. When not set, the Schema inherits the domain from the database it belongs to.",
      "$ref": "../../type/entityReference.json"
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/mlmodel.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/mlmodel.json
@ -265,6 +265,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of mlModel.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Domain the MLModel belongs to. When not set, the MLModel inherits the domain from the ML Model Service it belongs to.",
      "$ref": "../../type/entityReference.json"
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/topic.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/topic.json
@ -154,6 +154,10 @@
      "description": "Entity extension data with custom attributes added to the entity.",
      "$ref": "../../type/basic.json#/definitions/entityExtension"
    },
+    "sourceUrl": {
+      "description": "Source URL of topic.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
    "domain" : {
      "description": "Domain the Topic belongs to. When not set, the Topic inherits the domain from the messaging service it belongs to.",
      "$ref": "../../type/entityReference.json"