Added source url to search indexes and missing entites (#12773)

* added sourceUrl to missing entites and added sourceUrl to search  indexes

* edited desc in json files

* edited desc in json files

* working on adding api for getting entity from sourceUrl

* added API for getting Entity from sourceUrl

* added api to fetch entity based on sourceUrl

* remove desc

* working on adding alias

openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java

@@ -182,6 +182,26 @@ public class SearchResource {
     return searchClient.search(request);
   }
 
+  @GET
+  @Path("/sourceUrl")
+  @Operation(
+      operationId = "searchEntitiesWithSourceUrl",
+      summary = "Search entities",
+      responses = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "search response",
+            content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchResponse.class)))
+      })
+  public Response searchBySourceUrl(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(description = "source url") @QueryParam("sourceUrl") String sourceUrl)
+      throws IOException {
+
+    return searchClient.searchBySourceUrl(sourceUrl);
+  }
+
   @GET
   @Path("/suggest")
   @Operation(

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java

@@ -54,6 +54,8 @@ public interface SearchClient {
 
   Response search(SearchRequest request) throws IOException;
 
+  Response searchBySourceUrl(String sourceUrl) throws IOException;
+
   Response aggregate(String index, String fieldName, String value, String query) throws IOException;
 
   Response suggest(SearchRequest request) throws IOException;

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java

@@ -2,7 +2,10 @@ package org.openmetadata.service.search.elasticSearch;
 
 import static javax.ws.rs.core.Response.Status.OK;
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
-import static org.openmetadata.schema.type.EventType.*;
+import static org.openmetadata.schema.type.EventType.ENTITY_DELETED;
+import static org.openmetadata.schema.type.EventType.ENTITY_RESTORED;
+import static org.openmetadata.schema.type.EventType.ENTITY_SOFT_DELETED;
+import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED;
 import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
 import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
 import static org.openmetadata.service.Entity.FIELD_NAME;
@@ -52,6 +55,7 @@ import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -198,6 +202,14 @@ public class ElasticSearchClientImpl implements SearchClient {
         request.source(elasticSearchIndexMapping, XContentType.JSON);
         CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
         LOG.info("{} Created {}", elasticSearchIndexType.indexName, createIndexResponse.isAcknowledged());
+        // creating alias for indexes
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+        IndicesAliasesRequest.AliasActions aliasAction =
+            IndicesAliasesRequest.AliasActions.add()
+                .index(elasticSearchIndexType.indexName)
+                .alias("sourceUrlSearchAlias");
+        aliasesRequest.addAliasAction(aliasAction);
+        client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
       }
       elasticSearchIndexes.put(elasticSearchIndexType, IndexUtil.ElasticSearchIndexStatus.CREATED);
     } catch (Exception e) {
@@ -219,6 +231,14 @@ public class ElasticSearchClientImpl implements SearchClient {
       String elasticSearchIndexMapping = getIndexMapping(elasticSearchIndexType, lang);
       ENTITY_TO_MAPPING_SCHEMA_MAP.put(
           elasticSearchIndexType.entityType, JsonUtils.getMap(JsonUtils.readJson(elasticSearchIndexMapping)));
+      // creating alias for indexes
+      IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+      IndicesAliasesRequest.AliasActions aliasAction =
+          IndicesAliasesRequest.AliasActions.add()
+              .index(elasticSearchIndexType.indexName)
+              .alias("sourceUrlSearchAlias");
+      aliasesRequest.addAliasAction(aliasAction);
+      client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
       if (exists) {
         PutMappingRequest request = new PutMappingRequest(elasticSearchIndexType.indexName);
         request.source(elasticSearchIndexMapping, XContentType.JSON);
@@ -247,6 +267,14 @@ public class ElasticSearchClientImpl implements SearchClient {
       gRequest.local(false);
       boolean exists = client.indices().exists(gRequest, RequestOptions.DEFAULT);
       if (exists) {
+        // deleting alias for indexes
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+        IndicesAliasesRequest.AliasActions aliasAction =
+            IndicesAliasesRequest.AliasActions.remove()
+                .index(elasticSearchIndexType.indexName)
+                .alias("sourceUrlSearchAlias");
+        aliasesRequest.addAliasAction(aliasAction);
+        client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
         DeleteIndexRequest request = new DeleteIndexRequest(elasticSearchIndexType.indexName);
         AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
         LOG.info("{} Deleted {}", elasticSearchIndexType.indexName, deleteIndexResponse.isAcknowledged());
@@ -363,6 +391,22 @@ public class ElasticSearchClientImpl implements SearchClient {
     return Response.status(OK).entity(response).build();
   }
 
+  /**
+   * @param sourceUrl
+   * @return
+   */
+  @Override
+  public Response searchBySourceUrl(String sourceUrl) throws IOException {
+    QueryBuilder wildcardQuery = QueryBuilders.queryStringQuery(sourceUrl).field("sourceUrl").escape(true);
+    org.elasticsearch.action.search.SearchRequest searchRequest =
+        new org.elasticsearch.action.search.SearchRequest("sourceUrlSearchAlias");
+    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+    searchSourceBuilder.query(wildcardQuery);
+    searchRequest.source(searchSourceBuilder);
+    String response = client.search(searchRequest, RequestOptions.DEFAULT).toString();
+    return Response.status(OK).entity(response).build();
+  }
+
   @Override
   public Response aggregate(String index, String fieldName, String value, String query) throws IOException {
     SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java

@@ -2,7 +2,10 @@ package org.openmetadata.service.search.openSearch;
 
 import static javax.ws.rs.core.Response.Status.OK;
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
-import static org.openmetadata.schema.type.EventType.*;
+import static org.openmetadata.schema.type.EventType.ENTITY_DELETED;
+import static org.openmetadata.schema.type.EventType.ENTITY_RESTORED;
+import static org.openmetadata.schema.type.EventType.ENTITY_SOFT_DELETED;
+import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED;
 import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
 import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
 import static org.openmetadata.service.Entity.FIELD_NAME;
@@ -92,6 +95,7 @@ import org.openmetadata.service.search.indexes.TestCaseIndex;
 import org.openmetadata.service.search.indexes.UserIndex;
 import org.openmetadata.service.util.JsonUtils;
 import org.opensearch.OpenSearchException;
+import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.bulk.BulkItemResponse;
 import org.opensearch.action.bulk.BulkRequest;
@@ -189,6 +193,14 @@ public class OpenSearchClientImpl implements SearchClient {
         request.source(elasticSearchIndexMapping, XContentType.JSON);
         CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
         LOG.info("{} Created {}", elasticSearchIndexType.indexName, createIndexResponse.isAcknowledged());
+        // creating alias for indexes
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+        IndicesAliasesRequest.AliasActions aliasAction =
+            IndicesAliasesRequest.AliasActions.add()
+                .index(elasticSearchIndexType.indexName)
+                .alias("sourceUrlSearchAlias");
+        aliasesRequest.addAliasAction(aliasAction);
+        client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
       }
       elasticSearchIndexes.put(elasticSearchIndexType, IndexUtil.ElasticSearchIndexStatus.CREATED);
     } catch (Exception e) {
@@ -211,6 +223,14 @@ public class OpenSearchClientImpl implements SearchClient {
       String elasticSearchIndexMapping = getIndexMapping(elasticSearchIndexType, lang);
       ENTITY_TO_MAPPING_SCHEMA_MAP.put(
           elasticSearchIndexType.entityType, JsonUtils.getMap(JsonUtils.readJson(elasticSearchIndexMapping)));
+      // creating alias for indexes
+      IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+      IndicesAliasesRequest.AliasActions aliasAction =
+          IndicesAliasesRequest.AliasActions.add()
+              .index(elasticSearchIndexType.indexName)
+              .alias("sourceUrlSearchAlias");
+      aliasesRequest.addAliasAction(aliasAction);
+      client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
       if (exists) {
         PutMappingRequest request = new PutMappingRequest(elasticSearchIndexType.indexName);
         request.source(elasticSearchIndexMapping, XContentType.JSON);
@@ -239,6 +259,14 @@ public class OpenSearchClientImpl implements SearchClient {
       gRequest.local(false);
       boolean exists = client.indices().exists(gRequest, RequestOptions.DEFAULT);
       if (exists) {
+        // deleting alias for indexes
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
+        IndicesAliasesRequest.AliasActions aliasAction =
+            IndicesAliasesRequest.AliasActions.remove()
+                .index(elasticSearchIndexType.indexName)
+                .alias("sourceUrlSearchAlias");
+        aliasesRequest.addAliasAction(aliasAction);
+        client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
         DeleteIndexRequest request = new DeleteIndexRequest(elasticSearchIndexType.indexName);
         AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
         LOG.info("{} Deleted {}", elasticSearchIndexType.indexName, deleteIndexResponse.isAcknowledged());
@@ -357,6 +385,22 @@ public class OpenSearchClientImpl implements SearchClient {
     return Response.status(OK).entity(response).build();
   }
 
+  /**
+   * @param sourceUrl
+   * @return
+   */
+  @Override
+  public Response searchBySourceUrl(String sourceUrl) throws IOException {
+    QueryBuilder wildcardQuery = QueryBuilders.queryStringQuery(sourceUrl).field("sourceUrl").escape(true);
+    org.opensearch.action.search.SearchRequest searchRequest =
+        new org.opensearch.action.search.SearchRequest("sourceUrlSearchAlias");
+    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+    searchSourceBuilder.query(wildcardQuery);
+    searchRequest.source(searchSourceBuilder);
+    String response = client.search(searchRequest, RequestOptions.DEFAULT).toString();
+    return Response.status(OK).entity(response).build();
+  }
+
   public Response aggregate(String index, String fieldName, String value, String query) throws IOException {
     SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
     XContentParser filterParser =

openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json

@@ -87,6 +87,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "parent": {
         "properties": {
           "id": {

openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json

@@ -95,6 +95,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "columns": {
         "properties": {
           "name": {

openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json

@@ -91,6 +91,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "messageSchema": {
         "properties": {
           "schemaText": {

openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json

@@ -83,6 +83,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "parent": {
         "properties": {
           "id": {

openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json

@@ -105,6 +105,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "columns": {
         "properties": {
           "name": {

openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json

@@ -94,6 +94,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "messageSchema": {
         "properties": {
           "schemaType": {

openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json

@@ -78,6 +78,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "parent": {
         "properties": {
           "id": {

openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json

@@ -70,6 +70,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "columns": {
         "properties": {
           "name": {

openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json

@@ -63,6 +63,9 @@
       "href": {
         "type": "text"
       },
+      "sourceUrl": {
+        "type": "text"
+      },
       "messageSchema": {
         "properties": {
           "schemaType": {

openmetadata-spec/src/main/resources/json/schema/api/data/createContainer.json

@@ -72,6 +72,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of container.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Fully qualified name of the domain the Container belongs to.",
       "type": "string"

openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json

@@ -49,6 +49,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of database.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Fully qualified name of the domain the Database belongs to.",
       "type": "string"

openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json

@@ -45,6 +45,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of database schema.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Fully qualified name of the domain the Database Schema belongs to.",
       "type": "string"

openmetadata-spec/src/main/resources/json/schema/api/data/createMlModel.json

@@ -76,6 +76,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of mlModel.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Fully qualified name of the domain the MLModel belongs to.",
       "type": "string"

openmetadata-spec/src/main/resources/json/schema/api/data/createTopic.json

@@ -80,6 +80,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of topic.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Fully qualified name of the domain the Topic belongs to.",
       "type": "string",

openmetadata-spec/src/main/resources/json/schema/entity/data/container.json

@@ -168,6 +168,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of container.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Domain the Container belongs to. When not set, the Container inherits the domain from the storage service it belongs to.",
       "$ref": "../../type/entityReference.json"

openmetadata-spec/src/main/resources/json/schema/entity/data/database.json

@@ -108,6 +108,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of database.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Domain the Database belongs to. When not set, the Database inherits the domain from the database service it belongs to.",
       "$ref": "../../type/entityReference.json"

openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json

@@ -103,6 +103,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of database schema.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Domain the Database Schema belongs to. When not set, the Schema inherits the domain from the database it belongs to.",
       "$ref": "../../type/entityReference.json"

openmetadata-spec/src/main/resources/json/schema/entity/data/mlmodel.json

@@ -265,6 +265,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of mlModel.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Domain the MLModel belongs to. When not set, the MLModel inherits the domain from the ML Model Service it belongs to.",
       "$ref": "../../type/entityReference.json"

openmetadata-spec/src/main/resources/json/schema/entity/data/topic.json

@@ -154,6 +154,10 @@
       "description": "Entity extension data with custom attributes added to the entity.",
       "$ref": "../../type/basic.json#/definitions/entityExtension"
     },
+    "sourceUrl": {
+      "description": "Source URL of topic.",
+      "$ref": "../../type/basic.json#/definitions/sourceUrl"
+    },
     "domain" : {
       "description": "Domain the Topic belongs to. When not set, the Topic inherits the domain from the messaging service it belongs to.",
       "$ref": "../../type/entityReference.json"