yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-17 18:46:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade search versions (#13479)

Browse Source 
* working on maven shading

* update es in docker

* add maven shade for elasticsearch/opensearch dependencies

* add submodule for es/os dependencies

* upgrade docker elasticsearch to latest 8.10.2

* upgrade docker elasticsearch to latest 8.10.2

* Update Postgres dev to use OpenSearch

* Fix container service dependency

* revert conf change

---------

Co-authored-by: 07Himank <himank07mehta@gmail.com>
Co-authored-by: 07Himank <112613760+07Himank@users.noreply.github.com>
This commit is contained in:
Sriharsha Chintalapani 2023-10-08 11:55:34 -07:00 committed by GitHub
parent b06db047a4
commit 35c4e18663
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
57 changed files with 839 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
docker/development/docker-compose-postgres.yml
View File

@ -24,7 +24,7 @@ services:
restart: always restart: always
command: "--work_mem=10MB" command: "--work_mem=10MB"
depends_on: depends_on:
- elasticsearch - opensearch
environment: environment:
POSTGRES_USER: postgres POSTGRES_USER: postgres
POSTGRES_PASSWORD: password POSTGRES_PASSWORD: password
@ -42,12 +42,13 @@ services:
timeout: 10s timeout: 10s
retries: 10 retries: 10
elasticsearch: opensearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3 image: opensearchproject/opensearch:latest
container_name: openmetadata_elasticsearch container_name: openmetadata_opensearch
environment: environment:
- discovery.type=single-node - discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- plugins.security.disabled=true
networks: networks:
- local_app_net - local_app_net
expose: expose:
@ -57,7 +58,7 @@ services:
- "9200:9200" - "9200:9200"
- "9300:9300" - "9300:9300"
volumes: volumes:
- es-data:/usr/share/elasticsearch/data - es-data:/usr/share/opensearch/data
execute-migrate-all: execute-migrate-all:
build: build:
@ -113,7 +114,7 @@ services:
DB_PORT: ${DB_PORT:-5432} DB_PORT: ${DB_PORT:-5432}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db} OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations # ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch} ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- opensearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http} ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
@ -211,7 +212,7 @@ services:
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false} WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""} WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
depends_on: depends_on:
elasticsearch: opensearch:
condition: service_started condition: service_started
postgresql: postgresql:
condition: service_healthy condition: service_healthy
@ -344,7 +345,7 @@ services:
- "8586:8586" - "8586:8586"
- "5005:5005" - "5005:5005"
depends_on: depends_on:
elasticsearch: ingestion:
condition: service_started condition: service_started
postgresql: postgresql:
condition: service_healthy condition: service_healthy
@ -376,7 +377,7 @@ services:
command: command:
- "/opt/airflow/ingestion_dependency.sh" - "/opt/airflow/ingestion_dependency.sh"
depends_on: depends_on:
elasticsearch: opensearch:
condition: service_started condition: service_started
postgresql: postgresql:
condition: service_healthy condition: service_healthy

3
docker/development/docker-compose.yml
View File

@ -42,11 +42,12 @@ services:
- ./docker-volume/db-data:/var/lib/mysql - ./docker-volume/db-data:/var/lib/mysql
elasticsearch: elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3 image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
container_name: openmetadata_elasticsearch container_name: openmetadata_elasticsearch
environment: environment:
- discovery.type=single-node - discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- xpack.security.enabled=false
networks: networks:
- local_app_net - local_app_net
expose: expose:

3
docker/docker-compose-quickstart/docker-compose-postgres.yml
View File

@ -41,10 +41,11 @@ services:
elasticsearch: elasticsearch:
container_name: openmetadata_elasticsearch container_name: openmetadata_elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3 image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
environment: environment:
- discovery.type=single-node - discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- xpack.security.enabled=false
networks: networks:
- app_net - app_net
ports: ports:

3
docker/docker-compose-quickstart/docker-compose.yml
View File

@ -39,10 +39,11 @@ services:
elasticsearch: elasticsearch:
container_name: openmetadata_elasticsearch container_name: openmetadata_elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3 image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
environment: environment:
- discovery.type=single-node - discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- xpack.security.enabled=false
networks: networks:
- app_net - app_net
ports: ports:

208
openmetadata-service/pom.xml
View File

@ -39,6 +39,16 @@
<artifactId>common</artifactId> <artifactId>common</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.open-metadata</groupId>
<artifactId>elasticsearch-deps</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.open-metadata</groupId>
<artifactId>opensearch-deps</artifactId>
<version>${project.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.antlr</groupId> <groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId> <artifactId>antlr4-runtime</artifactId>
@ -129,57 +139,6 @@
<groupId>commons-io</groupId> <groupId>commons-io</groupId>
<artifactId>commons-io</artifactId> <artifactId>commons-io</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-grouping</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-memory</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial-extras</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial3d</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-suggest</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
</exclusion>
<exclusion>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-high-level-client</artifactId>
<version>${opensearch.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId> <artifactId>httpasyncclient</artifactId>
@ -769,6 +728,153 @@
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId> <artifactId>maven-source-plugin</artifactId>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.1</version> <!-- Use the latest version available -->
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactId>my-app-shaded</shadedArtifactId>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
<relocations>
<!-- Relocate Elasticsearch's Lucene -->
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-core</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-analyzers-common</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-backward-codecs</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-grouping</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-highlighter</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-join</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-memory</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-misc</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-queries</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-queryparser</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-sandbox</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-spatial3d</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-suggest</shadedPattern>
</relocation>
<!-- Relocate OpenSearch's Lucene -->
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-core</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-analysis-common</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-backward-codecs</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-grouping</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-highlighter</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-join</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-memory</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-misc</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-queries</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-queryparser</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-sandbox</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-spatial-extras</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-spatial3d</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-suggest</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>module-info.class</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<!-- add Main-Class to manifest file -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.openmetadata.service.OpenMetadataApplication</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins> </plugins>
</build> </build>
</project> </project>

4
openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
View File

@ -16,6 +16,8 @@ package org.openmetadata.service.resources.search;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.search.SearchRepository.ELASTIC_SEARCH_EXTENSION; import static org.openmetadata.service.search.SearchRepository.ELASTIC_SEARCH_EXTENSION;
import es.org.elasticsearch.action.search.SearchResponse;
import es.org.elasticsearch.search.suggest.Suggest;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Content;
@ -40,8 +42,6 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo; import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.suggest.Suggest;
import org.openmetadata.schema.api.CreateEventPublisherJob; import org.openmetadata.schema.api.CreateEventPublisherJob;
import org.openmetadata.schema.system.EventPublisherJob; import org.openmetadata.schema.system.EventPublisherJob;
import org.openmetadata.service.Entity; import org.openmetadata.service.Entity;

12
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
View File

@ -17,9 +17,9 @@ import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.exception.CustomExceptionMessage; import org.openmetadata.service.exception.CustomExceptionMessage;
import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.search.models.IndexMapping;
import org.openmetadata.service.util.SSLUtil; import org.openmetadata.service.util.SSLUtil;
import org.opensearch.action.bulk.BulkRequest; import os.org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.bulk.BulkResponse; import os.org.opensearch.action.bulk.BulkResponse;
import org.opensearch.client.RequestOptions; import os.org.opensearch.client.RequestOptions;
public interface SearchClient { public interface SearchClient {
@ -110,8 +110,8 @@ public interface SearchClient {
throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD); throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
} }
default org.elasticsearch.action.bulk.BulkResponse bulk( default es.org.elasticsearch.action.bulk.BulkResponse bulk(
org.elasticsearch.action.bulk.BulkRequest data, org.elasticsearch.client.RequestOptions options) es.org.elasticsearch.action.bulk.BulkRequest data, es.org.elasticsearch.client.RequestOptions options)
throws IOException { throws IOException {
throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD); throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
} }
@ -120,7 +120,7 @@ public interface SearchClient {
throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD); throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
} }
default int getSuccessFromBulkResponse(org.elasticsearch.action.bulk.BulkResponse response) { default int getSuccessFromBulkResponse(es.org.elasticsearch.action.bulk.BulkResponse response) {
throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD); throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
} }

180
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -19,6 +19,73 @@ import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED; import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED;
import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH; import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH;
import es.org.elasticsearch.action.ActionListener;
import es.org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import es.org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import es.org.elasticsearch.action.bulk.BulkItemResponse;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.bulk.BulkResponse;
import es.org.elasticsearch.action.delete.DeleteRequest;
import es.org.elasticsearch.action.delete.DeleteResponse;
import es.org.elasticsearch.action.search.SearchResponse;
import es.org.elasticsearch.action.support.WriteRequest;
import es.org.elasticsearch.action.support.master.AcknowledgedResponse;
import es.org.elasticsearch.action.update.UpdateRequest;
import es.org.elasticsearch.action.update.UpdateResponse;
import es.org.elasticsearch.client.RequestOptions;
import es.org.elasticsearch.client.RestClient;
import es.org.elasticsearch.client.RestClientBuilder;
import es.org.elasticsearch.client.RestHighLevelClient;
import es.org.elasticsearch.client.RestHighLevelClientBuilder;
import es.org.elasticsearch.client.indices.CreateIndexRequest;
import es.org.elasticsearch.client.indices.CreateIndexResponse;
import es.org.elasticsearch.client.indices.GetIndexRequest;
import es.org.elasticsearch.client.indices.PutMappingRequest;
import es.org.elasticsearch.common.lucene.search.function.CombineFunction;
import es.org.elasticsearch.common.settings.Settings;
import es.org.elasticsearch.common.unit.Fuzziness;
import es.org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import es.org.elasticsearch.core.TimeValue;
import es.org.elasticsearch.index.query.BoolQueryBuilder;
import es.org.elasticsearch.index.query.MatchQueryBuilder;
import es.org.elasticsearch.index.query.MultiMatchQueryBuilder;
import es.org.elasticsearch.index.query.Operator;
import es.org.elasticsearch.index.query.QueryBuilder;
import es.org.elasticsearch.index.query.QueryBuilders;
import es.org.elasticsearch.index.query.QueryStringQueryBuilder;
import es.org.elasticsearch.index.query.RangeQueryBuilder;
import es.org.elasticsearch.index.query.ScriptQueryBuilder;
import es.org.elasticsearch.index.query.TermQueryBuilder;
import es.org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionBuilder;
import es.org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import es.org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import es.org.elasticsearch.index.reindex.BulkByScrollResponse;
import es.org.elasticsearch.index.reindex.DeleteByQueryRequest;
import es.org.elasticsearch.index.reindex.UpdateByQueryRequest;
import es.org.elasticsearch.script.Script;
import es.org.elasticsearch.script.ScriptType;
import es.org.elasticsearch.search.SearchModule;
import es.org.elasticsearch.search.aggregations.AggregationBuilder;
import es.org.elasticsearch.search.aggregations.AggregationBuilders;
import es.org.elasticsearch.search.aggregations.BucketOrder;
import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import es.org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
import es.org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import es.org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import es.org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import es.org.elasticsearch.search.builder.SearchSourceBuilder;
import es.org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import es.org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import es.org.elasticsearch.search.sort.SortOrder;
import es.org.elasticsearch.search.suggest.Suggest;
import es.org.elasticsearch.search.suggest.SuggestBuilder;
import es.org.elasticsearch.search.suggest.SuggestBuilders;
import es.org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import es.org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
import es.org.elasticsearch.xcontent.NamedXContentRegistry;
import es.org.elasticsearch.xcontent.XContentParser;
import es.org.elasticsearch.xcontent.XContentType;
import java.io.IOException; import java.io.IOException;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
@ -41,72 +108,6 @@ import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider; import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.ScriptQueryBuilder;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionBuilder;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestBuilders;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentType;
import org.openmetadata.schema.DataInsightInterface; import org.openmetadata.schema.DataInsightInterface;
import org.openmetadata.schema.dataInsight.DataInsightChartResult; import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
@ -211,18 +212,6 @@ public class ElasticSearchClient implements SearchClient {
@Override @Override
public void createAliases(IndexMapping indexMapping) { public void createAliases(IndexMapping indexMapping) {
try { try {
ActionListener<AcknowledgedResponse> listener =
new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
LOG.debug("Created successfully: " + acknowledgedResponse.toString());
}
@Override
public void onFailure(Exception e) {
LOG.error("Creation failed: " + e.getMessage());
}
};
Set<String> aliases = new HashSet<>(indexMapping.getParentAliases()); Set<String> aliases = new HashSet<>(indexMapping.getParentAliases());
aliases.add(indexMapping.getAlias()); aliases.add(indexMapping.getAlias());
IndicesAliasesRequest.AliasActions aliasAction = IndicesAliasesRequest.AliasActions aliasAction =
@ -231,7 +220,7 @@ public class ElasticSearchClient implements SearchClient {
.aliases(aliases.toArray(new String[0])); .aliases(aliases.toArray(new String[0]));
IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest(); IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
aliasesRequest.addAliasAction(aliasAction); aliasesRequest.addAliasAction(aliasAction);
client.indices().updateAliasesAsync(aliasesRequest, RequestOptions.DEFAULT, listener); client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
} catch (Exception e) { } catch (Exception e) {
LOG.error(String.format("Failed to create alias for %s due to", indexMapping.getIndexName()), e); LOG.error(String.format("Failed to create alias for %s due to", indexMapping.getIndexName()), e);
} }
@ -384,7 +373,7 @@ public class ElasticSearchClient implements SearchClient {
String response = String response =
client client
.search( .search(
new org.elasticsearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder), new es.org.elasticsearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder),
RequestOptions.DEFAULT) RequestOptions.DEFAULT)
.toString(); .toString();
return Response.status(OK).entity(response).build(); return Response.status(OK).entity(response).build();
@ -392,8 +381,8 @@ public class ElasticSearchClient implements SearchClient {
@Override @Override
public Response searchBySourceUrl(String sourceUrl) throws IOException { public Response searchBySourceUrl(String sourceUrl) throws IOException {
org.elasticsearch.action.search.SearchRequest searchRequest = es.org.elasticsearch.action.search.SearchRequest searchRequest =
new org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("sourceUrl", sourceUrl))); searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("sourceUrl", sourceUrl)));
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
@ -403,8 +392,8 @@ public class ElasticSearchClient implements SearchClient {
@Override @Override
public Response searchByField(String fieldName, String fieldValue, String index) throws IOException { public Response searchByField(String fieldName, String fieldValue, String index) throws IOException {
org.elasticsearch.action.search.SearchRequest searchRequest = es.org.elasticsearch.action.search.SearchRequest searchRequest =
new org.elasticsearch.action.search.SearchRequest(index); new es.org.elasticsearch.action.search.SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.wildcardQuery(fieldName, fieldValue)); searchSourceBuilder.query(QueryBuilders.wildcardQuery(fieldName, fieldValue));
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
@ -433,7 +422,7 @@ public class ElasticSearchClient implements SearchClient {
String response = String response =
client client
.search( .search(
new org.elasticsearch.action.search.SearchRequest(index).source(searchSourceBuilder), new es.org.elasticsearch.action.search.SearchRequest(index).source(searchSourceBuilder),
RequestOptions.DEFAULT) RequestOptions.DEFAULT)
.toString(); .toString();
return Response.status(OK).entity(response).build(); return Response.status(OK).entity(response).build();
@ -463,8 +452,8 @@ public class ElasticSearchClient implements SearchClient {
.fetchSource( .fetchSource(
new FetchSourceContext( new FetchSourceContext(
request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {})); request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {}));
org.elasticsearch.action.search.SearchRequest searchRequest = es.org.elasticsearch.action.search.SearchRequest searchRequest =
new org.elasticsearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder); new es.org.elasticsearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Suggest suggest = searchResponse.getSuggest(); Suggest suggest = searchResponse.getSuggest();
return Response.status(OK).entity(suggest.toString()).build(); return Response.status(OK).entity(suggest.toString()).build();
@ -1119,7 +1108,7 @@ public class ElasticSearchClient implements SearchClient {
DataInsightChartResult.DataInsightChartType chartType, DataInsightChartResult.DataInsightChartType chartType,
String indexName) String indexName)
throws IOException, ParseException { throws IOException, ParseException {
org.elasticsearch.action.search.SearchRequest searchRequestTotalAssets = es.org.elasticsearch.action.search.SearchRequest searchRequestTotalAssets =
buildSearchRequest(scheduleTime, currentTime, null, team, chartType, null, null, null, indexName); buildSearchRequest(scheduleTime, currentTime, null, team, chartType, null, null, null, indexName);
SearchResponse searchResponseTotalAssets = client.search(searchRequestTotalAssets, RequestOptions.DEFAULT); SearchResponse searchResponseTotalAssets = client.search(searchRequestTotalAssets, RequestOptions.DEFAULT);
DataInsightChartResult processedDataTotalAssets = DataInsightChartResult processedDataTotalAssets =
@ -1150,7 +1139,7 @@ public class ElasticSearchClient implements SearchClient {
String queryFilter, String queryFilter,
String dataReportIndex) String dataReportIndex)
throws IOException, ParseException { throws IOException, ParseException {
org.elasticsearch.action.search.SearchRequest searchRequest = es.org.elasticsearch.action.search.SearchRequest searchRequest =
buildSearchRequest(startTs, endTs, tier, team, dataInsightChartName, size, from, queryFilter, dataReportIndex); buildSearchRequest(startTs, endTs, tier, team, dataInsightChartName, size, from, queryFilter, dataReportIndex);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return Response.status(OK).entity(processDataInsightChartResult(searchResponse, dataInsightChartName)).build(); return Response.status(OK).entity(processDataInsightChartResult(searchResponse, dataInsightChartName)).build();
@ -1203,7 +1192,7 @@ public class ElasticSearchClient implements SearchClient {
} }
} }
private static org.elasticsearch.action.search.SearchRequest buildSearchRequest( private static es.org.elasticsearch.action.search.SearchRequest buildSearchRequest(
Long startTs, Long startTs,
Long endTs, Long endTs,
String tier, String tier,
@ -1229,8 +1218,8 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder.sort("data.lifeCycle.accessed.timestamp", SortOrder.DESC); searchSourceBuilder.sort("data.lifeCycle.accessed.timestamp", SortOrder.DESC);
} }
org.elasticsearch.action.search.SearchRequest searchRequest = es.org.elasticsearch.action.search.SearchRequest searchRequest =
new org.elasticsearch.action.search.SearchRequest(dataReportIndex); new es.org.elasticsearch.action.search.SearchRequest(dataReportIndex);
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
return searchRequest; return searchRequest;
} }
@ -1478,6 +1467,10 @@ public class ElasticSearchClient implements SearchClient {
try { try {
RestClientBuilder restClientBuilder = RestClientBuilder restClientBuilder =
RestClient.builder(new HttpHost(esConfig.getHost(), esConfig.getPort(), esConfig.getScheme())); RestClient.builder(new HttpHost(esConfig.getHost(), esConfig.getPort(), esConfig.getScheme()));
RestClient restClient =
RestClient.builder(new HttpHost(esConfig.getHost(), esConfig.getPort(), esConfig.getScheme())).build();
if (StringUtils.isNotEmpty(esConfig.getUsername()) && StringUtils.isNotEmpty(esConfig.getPassword())) { if (StringUtils.isNotEmpty(esConfig.getUsername()) && StringUtils.isNotEmpty(esConfig.getPassword())) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials( credentialsProvider.setCredentials(
@ -1502,7 +1495,8 @@ public class ElasticSearchClient implements SearchClient {
requestConfigBuilder requestConfigBuilder
.setConnectTimeout(esConfig.getConnectionTimeoutSecs() * 1000) .setConnectTimeout(esConfig.getConnectionTimeoutSecs() * 1000)
.setSocketTimeout(esConfig.getSocketTimeoutSecs() * 1000)); .setSocketTimeout(esConfig.getSocketTimeoutSecs() * 1000));
return new RestHighLevelClient(restClientBuilder); // return new RestHighLevelClient(restClientBuilder);
return new RestHighLevelClientBuilder(restClient).setApiCompatibilityMode(true).build();
} catch (Exception e) { } catch (Exception e) {
LOG.error("Failed to create elastic search client ", e); LOG.error("Failed to create elastic search client ", e);
return null; return null;

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java
View File

@ -3,12 +3,12 @@ package org.openmetadata.service.search.elasticsearch;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.update.UpdateRequest;
import es.org.elasticsearch.xcontent.XContentType;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.xcontent.XContentType;
import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.analytics.ReportData; import org.openmetadata.schema.analytics.ReportData;
import org.openmetadata.schema.system.StepStats; import org.openmetadata.schema.system.StepStats;

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java
View File

@ -3,13 +3,13 @@ package org.openmetadata.service.search.elasticsearch;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.update.UpdateRequest;
import es.org.elasticsearch.xcontent.XContentType;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.xcontent.XContentType;
import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.system.StepStats; import org.openmetadata.schema.system.StepStats;

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java
View File

@ -3,11 +3,11 @@ package org.openmetadata.service.search.elasticsearch;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getSuccessFromBulkResponseEs; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getSuccessFromBulkResponseEs;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.bulk.BulkResponse;
import es.org.elasticsearch.client.RequestOptions;
import java.util.Map; import java.util.Map;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.RequestOptions;
import org.openmetadata.schema.system.StepStats; import org.openmetadata.schema.system.StepStats;
import org.openmetadata.service.exception.SinkException; import org.openmetadata.service.exception.SinkException;
import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.SearchRepository;

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsCountAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsCountAggregator; import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsCountAggregator;
public class ElasticSearchAggregatedUnusedAssetsCountAggregator public class ElasticSearchAggregatedUnusedAssetsCountAggregator

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsSizeAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsSizeAggregator; import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsSizeAggregator;
public class ElasticSearchAggregatedUnusedAssetsSizeAggregator public class ElasticSearchAggregatedUnusedAssetsSizeAggregator

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsCountAggregator; import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsCountAggregator;
public class ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator public class ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsSizeAggregator; import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsSizeAggregator;
public class ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator public class ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator

4
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDailyActiveUsersAggregator.java
View File

@ -1,8 +1,8 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.openmetadata.service.dataInsight.DailyActiveUsersAggregator; import org.openmetadata.service.dataInsight.DailyActiveUsersAggregator;
public class ElasticSearchDailyActiveUsersAggregator public class ElasticSearchDailyActiveUsersAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchEntitiesDescriptionAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.EntitiesDescriptionAggregator; import org.openmetadata.service.dataInsight.EntitiesDescriptionAggregator;
public class ElasticSearchEntitiesDescriptionAggregator public class ElasticSearchEntitiesDescriptionAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchEntitiesOwnerAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.EntitiesOwnerAggregator; import org.openmetadata.service.dataInsight.EntitiesOwnerAggregator;
public class ElasticSearchEntitiesOwnerAggregator public class ElasticSearchEntitiesOwnerAggregator

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchMostActiveUsersAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Max;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Max;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.MostActiveUsersAggregator; import org.openmetadata.service.dataInsight.MostActiveUsersAggregator;
public class ElasticSearchMostActiveUsersAggregator public class ElasticSearchMostActiveUsersAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchMostViewedEntitiesAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.MostViewedEntitiesAggregator; import org.openmetadata.service.dataInsight.MostViewedEntitiesAggregator;
public class ElasticSearchMostViewedEntitiesAggregator public class ElasticSearchMostViewedEntitiesAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchPageViewsByEntitiesAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.PageViewsByEntitiesAggregator; import org.openmetadata.service.dataInsight.PageViewsByEntitiesAggregator;
public class ElasticSearchPageViewsByEntitiesAggregator public class ElasticSearchPageViewsByEntitiesAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchServicesDescriptionAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.ServicesDescriptionAggregator; import org.openmetadata.service.dataInsight.ServicesDescriptionAggregator;
public class ElasticSearchServicesDescriptionAggregator public class ElasticSearchServicesDescriptionAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchServicesOwnerAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.ServicesOwnerAggregator; import org.openmetadata.service.dataInsight.ServicesOwnerAggregator;
public class ElasticSearchServicesOwnerAggregator public class ElasticSearchServicesOwnerAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchTotalEntitiesAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.TotalEntitiesAggregator; import org.openmetadata.service.dataInsight.TotalEntitiesAggregator;
public class ElasticSearchTotalEntitiesAggregator public class ElasticSearchTotalEntitiesAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchTotalEntitiesByTierAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List; import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.TotalEntitiesByTierAggregator; import org.openmetadata.service.dataInsight.TotalEntitiesByTierAggregator;
public class ElasticSearchTotalEntitiesByTierAggregator public class ElasticSearchTotalEntitiesByTierAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchUnusedAssetsAggregator.java
View File

@ -1,8 +1,8 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators; package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import org.apache.lucene.search.TotalHits; import es.org.apache.lucene.search.TotalHits;
import org.elasticsearch.search.SearchHit; import es.org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits; import es.org.elasticsearch.search.SearchHits;
import org.openmetadata.service.dataInsight.UnusedAssetsAggregator; import org.openmetadata.service.dataInsight.UnusedAssetsAggregator;
public class ElasticSearchUnusedAssetsAggregator extends UnusedAssetsAggregator<SearchHits, SearchHit, TotalHits> { public class ElasticSearchUnusedAssetsAggregator extends UnusedAssetsAggregator<SearchHits, SearchHit, TotalHits> {

216
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -47,7 +47,6 @@ import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
import org.openmetadata.service.jdbi3.DataInsightChartRepository; import org.openmetadata.service.jdbi3.DataInsightChartRepository;
import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchClient;
import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.search.UpdateSearchEventsConstant;
import org.openmetadata.service.search.indexes.ContainerIndex; import org.openmetadata.service.search.indexes.ContainerIndex;
import org.openmetadata.service.search.indexes.DashboardDataModelIndex; import org.openmetadata.service.search.indexes.DashboardDataModelIndex;
import org.openmetadata.service.search.indexes.DashboardIndex; import org.openmetadata.service.search.indexes.DashboardIndex;
@ -80,72 +79,72 @@ import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSear
import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchTotalEntitiesByTierAggregator; import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchTotalEntitiesByTierAggregator;
import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchUnusedAssetsAggregator; import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchUnusedAssetsAggregator;
import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.JsonUtils;
import org.opensearch.action.ActionListener; import os.org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; import os.org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import os.org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
import org.opensearch.action.bulk.BulkItemResponse; import os.org.opensearch.action.bulk.BulkItemResponse;
import org.opensearch.action.bulk.BulkRequest; import os.org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.bulk.BulkResponse; import os.org.opensearch.action.bulk.BulkResponse;
import org.opensearch.action.delete.DeleteRequest; import os.org.opensearch.action.delete.DeleteRequest;
import org.opensearch.action.delete.DeleteResponse; import os.org.opensearch.action.delete.DeleteResponse;
import org.opensearch.action.search.SearchResponse; import os.org.opensearch.action.search.SearchResponse;
import org.opensearch.action.support.WriteRequest; import os.org.opensearch.action.support.WriteRequest;
import org.opensearch.action.support.master.AcknowledgedResponse; import os.org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.action.update.UpdateRequest; import os.org.opensearch.action.update.UpdateRequest;
import org.opensearch.action.update.UpdateResponse; import os.org.opensearch.action.update.UpdateResponse;
import org.opensearch.client.RequestOptions; import os.org.opensearch.client.RequestOptions;
import org.opensearch.client.RestClient; import os.org.opensearch.client.RestClient;
import org.opensearch.client.RestClientBuilder; import os.org.opensearch.client.RestClientBuilder;
import org.opensearch.client.RestHighLevelClient; import os.org.opensearch.client.RestHighLevelClient;
import org.opensearch.client.indices.CreateIndexRequest; import os.org.opensearch.client.indices.CreateIndexRequest;
import org.opensearch.client.indices.CreateIndexResponse; import os.org.opensearch.client.indices.CreateIndexResponse;
import org.opensearch.client.indices.GetIndexRequest; import os.org.opensearch.client.indices.GetIndexRequest;
import org.opensearch.client.indices.PutMappingRequest; import os.org.opensearch.client.indices.PutMappingRequest;
import org.opensearch.common.lucene.search.function.CombineFunction; import os.org.opensearch.common.lucene.search.function.CombineFunction;
import org.opensearch.common.settings.Settings; import os.org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.Fuzziness; import os.org.opensearch.common.unit.Fuzziness;
import org.opensearch.common.unit.TimeValue; import os.org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.LoggingDeprecationHandler; import os.org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.NamedXContentRegistry; import os.org.opensearch.common.xcontent.NamedXContentRegistry;
import org.opensearch.common.xcontent.XContentParser; import os.org.opensearch.common.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType; import os.org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.query.BoolQueryBuilder; import os.org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.MatchQueryBuilder; import os.org.opensearch.index.query.MatchQueryBuilder;
import org.opensearch.index.query.MultiMatchQueryBuilder; import os.org.opensearch.index.query.MultiMatchQueryBuilder;
import org.opensearch.index.query.Operator; import os.org.opensearch.index.query.Operator;
import org.opensearch.index.query.QueryBuilder; import os.org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders; import os.org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.QueryStringQueryBuilder; import os.org.opensearch.index.query.QueryStringQueryBuilder;
import org.opensearch.index.query.RangeQueryBuilder; import os.org.opensearch.index.query.RangeQueryBuilder;
import org.opensearch.index.query.ScriptQueryBuilder; import os.org.opensearch.index.query.ScriptQueryBuilder;
import org.opensearch.index.query.TermQueryBuilder; import os.org.opensearch.index.query.TermQueryBuilder;
import org.opensearch.index.query.functionscore.FieldValueFactorFunctionBuilder; import os.org.opensearch.index.query.functionscore.FieldValueFactorFunctionBuilder;
import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; import os.org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.opensearch.index.query.functionscore.ScoreFunctionBuilders; import os.org.opensearch.index.query.functionscore.ScoreFunctionBuilders;
import org.opensearch.index.reindex.BulkByScrollResponse; import os.org.opensearch.index.reindex.BulkByScrollResponse;
import org.opensearch.index.reindex.DeleteByQueryRequest; import os.org.opensearch.index.reindex.DeleteByQueryRequest;
import org.opensearch.index.reindex.UpdateByQueryRequest; import os.org.opensearch.index.reindex.UpdateByQueryRequest;
import org.opensearch.script.Script; import os.org.opensearch.script.Script;
import org.opensearch.script.ScriptType; import os.org.opensearch.script.ScriptType;
import org.opensearch.search.SearchModule; import os.org.opensearch.search.SearchModule;
import org.opensearch.search.aggregations.AggregationBuilder; import os.org.opensearch.search.aggregations.AggregationBuilder;
import org.opensearch.search.aggregations.AggregationBuilders; import os.org.opensearch.search.aggregations.AggregationBuilders;
import org.opensearch.search.aggregations.BucketOrder; import os.org.opensearch.search.aggregations.BucketOrder;
import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.opensearch.search.aggregations.bucket.terms.IncludeExclude; import os.org.opensearch.search.aggregations.bucket.terms.IncludeExclude;
import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import os.org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; import os.org.opensearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import os.org.opensearch.search.aggregations.metrics.SumAggregationBuilder;
import org.opensearch.search.builder.SearchSourceBuilder; import os.org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.search.fetch.subphase.FetchSourceContext; import os.org.opensearch.search.fetch.subphase.FetchSourceContext;
import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import os.org.opensearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.opensearch.search.sort.SortOrder; import os.org.opensearch.search.sort.SortOrder;
import org.opensearch.search.suggest.Suggest; import os.org.opensearch.search.suggest.Suggest;
import org.opensearch.search.suggest.SuggestBuilder; import os.org.opensearch.search.suggest.SuggestBuilder;
import org.opensearch.search.suggest.SuggestBuilders; import os.org.opensearch.search.suggest.SuggestBuilders;
import org.opensearch.search.suggest.completion.CompletionSuggestionBuilder; import os.org.opensearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.opensearch.search.suggest.completion.context.CategoryQueryContext; import os.org.opensearch.search.suggest.completion.context.CategoryQueryContext;
@Slf4j @Slf4j
// Not tagged with Repository annotation as it is programmatically initialized // Not tagged with Repository annotation as it is programmatically initialized
@ -155,7 +154,7 @@ public class OpenSearchClient implements SearchClient {
private final boolean isClientAvailable; private final boolean isClientAvailable;
static { static {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, List.of()); SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of());
X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents()); X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents());
} }
@ -192,13 +191,13 @@ public class OpenSearchClient implements SearchClient {
// creating alias for indexes // creating alias for indexes
createAliases(indexMapping); createAliases(indexMapping);
} catch (Exception e) { } catch (Exception e) {
LOG.error("Failed to create Elastic Search indexes due to", e); LOG.error("Failed to create Open Search indexes due to", e);
return false; return false;
} }
return true; return true;
} else { } else {
LOG.error( LOG.error(
"Failed to create Elastic Search index as client is not property configured, Please check your OpenMetadata configuration"); "Failed to create Open Search index as client is not property configured, Please check your OpenMetadata configuration");
return false; return false;
} }
} }
@ -206,27 +205,15 @@ public class OpenSearchClient implements SearchClient {
@Override @Override
public void createAliases(IndexMapping indexMapping) { public void createAliases(IndexMapping indexMapping) {
try { try {
ActionListener<AcknowledgedResponse> listener =
new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
LOG.debug("Created successfully: " + acknowledgedResponse.toString());
}
@Override
public void onFailure(Exception e) {
LOG.error("Creation failed: " + e.getMessage());
}
};
List<String> aliases = indexMapping.getParentAliases(); List<String> aliases = indexMapping.getParentAliases();
aliases.add(indexMapping.getAlias()); aliases.add(indexMapping.getAlias());
IndicesAliasesRequest.AliasActions aliasAction = IndicesAliasesRequest.AliasActions aliasAction =
IndicesAliasesRequest.AliasActions.add() IndicesAliasesRequest.AliasActions.add()
.index(indexMapping.getIndexMappingFile()) .index(indexMapping.getIndexName())
.aliases(aliases.toArray(new String[0])); .aliases(aliases.toArray(new String[0]));
IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest(); IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
aliasesRequest.addAliasAction(aliasAction); aliasesRequest.addAliasAction(aliasAction);
client.indices().updateAliasesAsync(aliasesRequest, RequestOptions.DEFAULT, listener); client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
} catch (Exception e) { } catch (Exception e) {
LOG.error(String.format("Failed to create alias for %s due to", indexMapping.getIndexName()), e); LOG.error(String.format("Failed to create alias for %s due to", indexMapping.getIndexName()), e);
} }
@ -240,7 +227,7 @@ public class OpenSearchClient implements SearchClient {
AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
LOG.debug("{} Updated {}", indexMapping.getIndexMappingFile(), putMappingResponse.isAcknowledged()); LOG.debug("{} Updated {}", indexMapping.getIndexMappingFile(), putMappingResponse.isAcknowledged());
} catch (Exception e) { } catch (Exception e) {
LOG.error(String.format("Failed to Update Elastic Search index %s due to", indexMapping.getIndexName()), e); LOG.error(String.format("Failed to Update Open Search index %s due to", indexMapping.getIndexName()), e);
} }
} }
@ -251,7 +238,7 @@ public class OpenSearchClient implements SearchClient {
AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT); AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
LOG.debug("{} Deleted {}", indexMapping.getIndexName(), deleteIndexResponse.isAcknowledged()); LOG.debug("{} Deleted {}", indexMapping.getIndexName(), deleteIndexResponse.isAcknowledged());
} catch (IOException e) { } catch (IOException e) {
LOG.error("Failed to delete Elastic Search indexes due to", e); LOG.error("Failed to delete Open Search indexes due to", e);
} }
} }
@ -359,11 +346,11 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder.sort(request.getSortFieldParam(), SortOrder.fromString(request.getSortOrder())); searchSourceBuilder.sort(request.getSortFieldParam(), SortOrder.fromString(request.getSortOrder()));
} }
/* for performance reasons ElasticSearch doesn't provide accurate hits /* for performance reasons OpenSearch doesn't provide accurate hits
if we enable trackTotalHits parameter it will try to match every result, count and return hits if we enable trackTotalHits parameter it will try to match every result, count and return hits
however in most cases for search results an approximate value is good enough. however in most cases for search results an approximate value is good enough.
we are displaying total entity counts in landing page and explore page where we need the total count we are displaying total entity counts in landing page and explore page where we need the total count
https://github.com/elastic/elasticsearch/issues/33028 */ https://github.com/Open/Opensearch/issues/33028 */
searchSourceBuilder.fetchSource( searchSourceBuilder.fetchSource(
new FetchSourceContext( new FetchSourceContext(
request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {})); request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {}));
@ -378,7 +365,7 @@ public class OpenSearchClient implements SearchClient {
String response = String response =
client client
.search( .search(
new org.opensearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder), new os.org.opensearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder),
RequestOptions.DEFAULT) RequestOptions.DEFAULT)
.toString(); .toString();
return Response.status(OK).entity(response).build(); return Response.status(OK).entity(response).build();
@ -387,8 +374,8 @@ public class OpenSearchClient implements SearchClient {
@Override @Override
public Response searchBySourceUrl(String sourceUrl) throws IOException { public Response searchBySourceUrl(String sourceUrl) throws IOException {
QueryBuilder wildcardQuery = QueryBuilders.queryStringQuery(sourceUrl).field("sourceUrl").escape(true); QueryBuilder wildcardQuery = QueryBuilders.queryStringQuery(sourceUrl).field("sourceUrl").escape(true);
org.opensearch.action.search.SearchRequest searchRequest = os.org.opensearch.action.search.SearchRequest searchRequest =
new org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS); new os.org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(wildcardQuery); searchSourceBuilder.query(wildcardQuery);
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
@ -398,7 +385,8 @@ public class OpenSearchClient implements SearchClient {
@Override @Override
public Response searchByField(String fieldName, String fieldValue, String index) throws IOException { public Response searchByField(String fieldName, String fieldValue, String index) throws IOException {
org.opensearch.action.search.SearchRequest searchRequest = new org.opensearch.action.search.SearchRequest(index); os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.wildcardQuery(fieldName, fieldValue)); searchSourceBuilder.query(QueryBuilders.wildcardQuery(fieldName, fieldValue));
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
@ -426,7 +414,7 @@ public class OpenSearchClient implements SearchClient {
String response = String response =
client client
.search( .search(
new org.opensearch.action.search.SearchRequest(index).source(searchSourceBuilder), new os.org.opensearch.action.search.SearchRequest(index).source(searchSourceBuilder),
RequestOptions.DEFAULT) RequestOptions.DEFAULT)
.toString(); .toString();
return Response.status(OK).entity(response).build(); return Response.status(OK).entity(response).build();
@ -441,12 +429,12 @@ public class OpenSearchClient implements SearchClient {
new ActionListener<>() { new ActionListener<>() {
@Override @Override
public void onResponse(UpdateResponse updateResponse) { public void onResponse(UpdateResponse updateResponse) {
LOG.debug("Created successfully: " + updateResponse.toString()); LOG.debug("Entity Updated successfully: " + updateResponse.toString());
} }
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
LOG.error("Creation failed: " + e.getMessage()); LOG.error("Entity Update failed: " + e.getMessage());
} }
}; };
client.updateAsync(updateRequest, RequestOptions.DEFAULT, listener); client.updateAsync(updateRequest, RequestOptions.DEFAULT, listener);
@ -476,8 +464,8 @@ public class OpenSearchClient implements SearchClient {
.fetchSource( .fetchSource(
new FetchSourceContext( new FetchSourceContext(
request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {})); request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {}));
org.opensearch.action.search.SearchRequest searchRequest = os.org.opensearch.action.search.SearchRequest searchRequest =
new org.opensearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder); new os.org.opensearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Suggest suggest = searchResponse.getSuggest(); Suggest suggest = searchResponse.getSuggest();
return Response.status(OK).entity(suggest.toString()).build(); return Response.status(OK).entity(suggest.toString()).build();
@ -932,7 +920,7 @@ public class OpenSearchClient implements SearchClient {
ScriptQueryBuilder scriptQuery = new ScriptQueryBuilder(script); ScriptQueryBuilder scriptQuery = new ScriptQueryBuilder(script);
DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexName); DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexName);
deleteByQueryRequest.setQuery(scriptQuery); deleteByQueryRequest.setQuery(scriptQuery);
deleteEntityFromElasticSearchByQuery(deleteByQueryRequest); deleteEntityFromOpenSearchByQuery(deleteByQueryRequest);
} }
} }
@ -940,7 +928,7 @@ public class OpenSearchClient implements SearchClient {
public void deleteEntity(String indexName, String docId) { public void deleteEntity(String indexName, String docId) {
if (isClientAvailable) { if (isClientAvailable) {
DeleteRequest deleteRequest = new DeleteRequest(indexName, docId); DeleteRequest deleteRequest = new DeleteRequest(indexName, docId);
deleteEntityFromElasticSearch(deleteRequest); deleteEntityFromOpenSearch(deleteRequest);
} }
} }
@ -953,7 +941,7 @@ public class OpenSearchClient implements SearchClient {
queryBuilder.must(new TermQueryBuilder(p.getKey(), p.getValue())); queryBuilder.must(new TermQueryBuilder(p.getKey(), p.getValue()));
} }
deleteByQueryRequest.setQuery(queryBuilder); deleteByQueryRequest.setQuery(queryBuilder);
deleteEntityFromElasticSearchByQuery(deleteByQueryRequest); deleteEntityFromOpenSearchByQuery(deleteByQueryRequest);
} }
} }
@ -979,7 +967,7 @@ public class OpenSearchClient implements SearchClient {
updateByQueryRequest.setQuery(queryBuilder); updateByQueryRequest.setQuery(queryBuilder);
Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptTxt, new HashMap<>()); Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptTxt, new HashMap<>());
updateByQueryRequest.setScript(script); updateByQueryRequest.setScript(script);
updateElasticSearchByQuery(updateByQueryRequest); updateOpenSearchByQuery(updateByQueryRequest);
} }
} }
@ -1007,11 +995,11 @@ public class OpenSearchClient implements SearchClient {
updates.getKey(), updates.getKey(),
JsonUtils.getMap(updates.getValue() == null ? new HashMap<>() : updates.getValue())); JsonUtils.getMap(updates.getValue() == null ? new HashMap<>() : updates.getValue()));
updateByQueryRequest.setScript(script); updateByQueryRequest.setScript(script);
updateElasticSearchByQuery(updateByQueryRequest); updateOpenSearchByQuery(updateByQueryRequest);
} }
} }
private void updateElasticSearchByQuery(UpdateByQueryRequest updateByQueryRequest) { private void updateOpenSearchByQuery(UpdateByQueryRequest updateByQueryRequest) {
if (updateByQueryRequest != null && isClientAvailable) { if (updateByQueryRequest != null && isClientAvailable) {
updateByQueryRequest.setRefresh(true); updateByQueryRequest.setRefresh(true);
LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest); LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest);
@ -1034,26 +1022,26 @@ public class OpenSearchClient implements SearchClient {
public void updateOpenSearch(UpdateRequest updateRequest) { public void updateOpenSearch(UpdateRequest updateRequest) {
if (updateRequest != null && isClientAvailable) { if (updateRequest != null && isClientAvailable) {
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
LOG.debug(UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH, updateRequest); LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateRequest);
ActionListener<UpdateResponse> listener = ActionListener<UpdateResponse> listener =
new ActionListener<>() { new ActionListener<>() {
@Override @Override
public void onResponse(UpdateResponse updateResponse) { public void onResponse(UpdateResponse updateResponse) {
LOG.debug("Created successfully: " + updateResponse.toString()); LOG.debug("Entity Updated successfully: " + updateResponse.toString());
} }
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
LOG.error("Creation failed: " + e.getMessage()); LOG.error("Entity Update failed: " + e.getMessage());
} }
}; };
client.updateAsync(updateRequest, RequestOptions.DEFAULT, listener); client.updateAsync(updateRequest, RequestOptions.DEFAULT, listener);
} }
} }
private void deleteEntityFromElasticSearch(DeleteRequest deleteRequest) { private void deleteEntityFromOpenSearch(DeleteRequest deleteRequest) {
if (deleteRequest != null && isClientAvailable) { if (deleteRequest != null && isClientAvailable) {
LOG.debug(UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH, deleteRequest); LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, deleteRequest);
ActionListener<DeleteResponse> listener = ActionListener<DeleteResponse> listener =
new ActionListener<>() { new ActionListener<>() {
@Override @Override
@ -1071,7 +1059,7 @@ public class OpenSearchClient implements SearchClient {
} }
} }
private void deleteEntityFromElasticSearchByQuery(DeleteByQueryRequest deleteRequest) { private void deleteEntityFromOpenSearchByQuery(DeleteByQueryRequest deleteRequest) {
if (deleteRequest != null && isClientAvailable) { if (deleteRequest != null && isClientAvailable) {
deleteRequest.setRefresh(true); deleteRequest.setRefresh(true);
ActionListener<BulkByScrollResponse> listener = ActionListener<BulkByScrollResponse> listener =
@ -1124,7 +1112,7 @@ public class OpenSearchClient implements SearchClient {
DataInsightChartResult.DataInsightChartType chartType, DataInsightChartResult.DataInsightChartType chartType,
String indexName) String indexName)
throws IOException, ParseException { throws IOException, ParseException {
org.opensearch.action.search.SearchRequest searchRequestTotalAssets = os.org.opensearch.action.search.SearchRequest searchRequestTotalAssets =
buildSearchRequest(scheduleTime, currentTime, null, team, chartType, null, null, null, indexName); buildSearchRequest(scheduleTime, currentTime, null, team, chartType, null, null, null, indexName);
SearchResponse searchResponseTotalAssets = client.search(searchRequestTotalAssets, RequestOptions.DEFAULT); SearchResponse searchResponseTotalAssets = client.search(searchRequestTotalAssets, RequestOptions.DEFAULT);
DataInsightChartResult processedDataTotalAssets = DataInsightChartResult processedDataTotalAssets =
@ -1155,7 +1143,7 @@ public class OpenSearchClient implements SearchClient {
String queryFilter, String queryFilter,
String dataReportIndex) String dataReportIndex)
throws IOException, ParseException { throws IOException, ParseException {
org.opensearch.action.search.SearchRequest searchRequest = os.org.opensearch.action.search.SearchRequest searchRequest =
buildSearchRequest(startTs, endTs, tier, team, dataInsightChartName, size, from, queryFilter, dataReportIndex); buildSearchRequest(startTs, endTs, tier, team, dataInsightChartName, size, from, queryFilter, dataReportIndex);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return Response.status(OK).entity(processDataInsightChartResult(searchResponse, dataInsightChartName)).build(); return Response.status(OK).entity(processDataInsightChartResult(searchResponse, dataInsightChartName)).build();
@ -1208,7 +1196,7 @@ public class OpenSearchClient implements SearchClient {
} }
} }
private static org.opensearch.action.search.SearchRequest buildSearchRequest( private static os.org.opensearch.action.search.SearchRequest buildSearchRequest(
Long startTs, Long startTs,
Long endTs, Long endTs,
String tier, String tier,
@ -1233,8 +1221,8 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder.sort("data.lifeCycle.accessed.timestamp", SortOrder.DESC); searchSourceBuilder.sort("data.lifeCycle.accessed.timestamp", SortOrder.DESC);
} }
org.opensearch.action.search.SearchRequest searchRequest = os.org.opensearch.action.search.SearchRequest searchRequest =
new org.opensearch.action.search.SearchRequest(dataReportIndex); new os.org.opensearch.action.search.SearchRequest(dataReportIndex);
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
return searchRequest; return searchRequest;
} }

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java
View File

@ -16,9 +16,9 @@ import org.openmetadata.service.search.models.IndexMapping;
import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.workflows.interfaces.Processor; import org.openmetadata.service.workflows.interfaces.Processor;
import org.opensearch.action.bulk.BulkRequest; import os.org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.update.UpdateRequest; import os.org.opensearch.action.update.UpdateRequest;
import org.opensearch.common.xcontent.XContentType; import os.org.opensearch.common.xcontent.XContentType;
@Slf4j @Slf4j
public class OpenSearchDataInsightProcessor implements Processor<BulkRequest, ResultList<ReportData>> { public class OpenSearchDataInsightProcessor implements Processor<BulkRequest, ResultList<ReportData>> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java
View File

@ -17,9 +17,9 @@ import org.openmetadata.service.search.models.IndexMapping;
import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.workflows.interfaces.Processor; import org.openmetadata.service.workflows.interfaces.Processor;
import org.opensearch.action.bulk.BulkRequest; import os.org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.update.UpdateRequest; import os.org.opensearch.action.update.UpdateRequest;
import org.opensearch.common.xcontent.XContentType; import os.org.opensearch.common.xcontent.XContentType;
@Slf4j @Slf4j
public class OpenSearchEntitiesProcessor implements Processor<BulkRequest, ResultList<? extends EntityInterface>> { public class OpenSearchEntitiesProcessor implements Processor<BulkRequest, ResultList<? extends EntityInterface>> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexSink.java
View File

@ -9,9 +9,9 @@ import org.openmetadata.schema.system.StepStats;
import org.openmetadata.service.exception.SinkException; import org.openmetadata.service.exception.SinkException;
import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.SearchRepository;
import org.openmetadata.service.workflows.interfaces.Sink; import org.openmetadata.service.workflows.interfaces.Sink;
import org.opensearch.action.bulk.BulkRequest; import os.org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.bulk.BulkResponse; import os.org.opensearch.action.bulk.BulkResponse;
import org.opensearch.client.RequestOptions; import os.org.opensearch.client.RequestOptions;
@Slf4j @Slf4j
public class OpenSearchIndexSink implements Sink<BulkRequest, BulkResponse> { public class OpenSearchIndexSink implements Sink<BulkRequest, BulkResponse> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsCountAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsCountAggregator; import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsCountAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchAggregatedUnusedAssetsCountAggregator public class OpenSearchAggregatedUnusedAssetsCountAggregator
extends AggregatedUnusedAssetsCountAggregator<Aggregations, Histogram, Bucket, Sum> { extends AggregatedUnusedAssetsCountAggregator<Aggregations, Histogram, Bucket, Sum> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsSizeAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsSizeAggregator; import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsSizeAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchAggregatedUnusedAssetsSizeAggregator public class OpenSearchAggregatedUnusedAssetsSizeAggregator
extends AggregatedUnusedAssetsSizeAggregator<Aggregations, Histogram, Bucket, Sum> { extends AggregatedUnusedAssetsSizeAggregator<Aggregations, Histogram, Bucket, Sum> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsCountAggregator; import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsCountAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator public class OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator
extends AggregatedUsedvsUnusedAssetsCountAggregator<Aggregations, Histogram, Bucket, Sum> { extends AggregatedUsedvsUnusedAssetsCountAggregator<Aggregations, Histogram, Bucket, Sum> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsSizeAggregator; import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsSizeAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator public class OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator
extends AggregatedUsedvsUnusedAssetsSizeAggregator<Aggregations, Histogram, Bucket, Sum> { extends AggregatedUsedvsUnusedAssetsSizeAggregator<Aggregations, Histogram, Bucket, Sum> {

4
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDailyActiveUsersAggregator.java
View File

@ -1,8 +1,8 @@
package org.openmetadata.service.search.opensearch.dataInsightAggregator; package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram; import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
public class OpenSearchDailyActiveUsersAggregator public class OpenSearchDailyActiveUsersAggregator
extends org.openmetadata.service.dataInsight.DailyActiveUsersAggregator<Aggregations, Histogram, Histogram.Bucket> { extends org.openmetadata.service.dataInsight.DailyActiveUsersAggregator<Aggregations, Histogram, Histogram.Bucket> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchEntitiesDescriptionAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.EntitiesDescriptionAggregator; import org.openmetadata.service.dataInsight.EntitiesDescriptionAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchEntitiesDescriptionAggregator public class OpenSearchEntitiesDescriptionAggregator
extends EntitiesDescriptionAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> { extends EntitiesDescriptionAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchEntitiesOwnerAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.EntitiesOwnerAggregator; import org.openmetadata.service.dataInsight.EntitiesOwnerAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchEntitiesOwnerAggregator public class OpenSearchEntitiesOwnerAggregator
extends EntitiesOwnerAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> { extends EntitiesOwnerAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchMostActiveUsersAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.MostActiveUsersAggregator; import org.openmetadata.service.dataInsight.MostActiveUsersAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Max; import os.org.opensearch.search.aggregations.metrics.Max;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchMostActiveUsersAggregator public class OpenSearchMostActiveUsersAggregator
extends MostActiveUsersAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum, Max> { extends MostActiveUsersAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum, Max> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchMostViewedEntitiesAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.MostViewedEntitiesAggregator; import org.openmetadata.service.dataInsight.MostViewedEntitiesAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchMostViewedEntitiesAggregator public class OpenSearchMostViewedEntitiesAggregator
extends MostViewedEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> { extends MostViewedEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchPageViewsByEntitiesAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.PageViewsByEntitiesAggregator; import org.openmetadata.service.dataInsight.PageViewsByEntitiesAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchPageViewsByEntitiesAggregator public class OpenSearchPageViewsByEntitiesAggregator
extends PageViewsByEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> { extends PageViewsByEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchServicesDescriptionAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.ServicesDescriptionAggregator; import org.openmetadata.service.dataInsight.ServicesDescriptionAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchServicesDescriptionAggregator public class OpenSearchServicesDescriptionAggregator
extends ServicesDescriptionAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> { extends ServicesDescriptionAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchServicesOwnerAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.ServicesOwnerAggregator; import org.openmetadata.service.dataInsight.ServicesOwnerAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchServicesOwnerAggregator public class OpenSearchServicesOwnerAggregator
extends ServicesOwnerAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> { extends ServicesOwnerAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchTotalEntitiesAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.TotalEntitiesAggregator; import org.openmetadata.service.dataInsight.TotalEntitiesAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchTotalEntitiesAggregator public class OpenSearchTotalEntitiesAggregator
extends TotalEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> { extends TotalEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchTotalEntitiesByTierAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List; import java.util.List;
import org.openmetadata.service.dataInsight.TotalEntitiesByTierAggregator; import org.openmetadata.service.dataInsight.TotalEntitiesByTierAggregator;
import org.opensearch.search.aggregations.Aggregations; import os.org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum; import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchTotalEntitiesByTierAggregator public class OpenSearchTotalEntitiesByTierAggregator
extends TotalEntitiesByTierAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> { extends TotalEntitiesByTierAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchUnusedAssetsAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.opensearch.dataInsightAggregator; package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import org.apache.lucene.search.TotalHits;
import org.openmetadata.service.dataInsight.UnusedAssetsAggregator; import org.openmetadata.service.dataInsight.UnusedAssetsAggregator;
import org.opensearch.search.SearchHit; import os.org.apache.lucene.search.TotalHits;
import org.opensearch.search.SearchHits; import os.org.opensearch.search.SearchHit;
import os.org.opensearch.search.SearchHits;
public class OpenSearchUnusedAssetsAggregator extends UnusedAssetsAggregator<SearchHits, SearchHit, TotalHits> { public class OpenSearchUnusedAssetsAggregator extends UnusedAssetsAggregator<SearchHits, SearchHit, TotalHits> {
public OpenSearchUnusedAssetsAggregator(SearchHits hits) { public OpenSearchUnusedAssetsAggregator(SearchHits hits) {

8
openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java
View File

@ -18,8 +18,8 @@ import org.openmetadata.schema.system.StepStats;
import org.openmetadata.service.Entity; import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.EntityRepository;
import org.opensearch.action.bulk.BulkItemResponse; import os.org.opensearch.action.bulk.BulkItemResponse;
import org.opensearch.action.bulk.BulkResponse; import os.org.opensearch.action.bulk.BulkResponse;
public class ReindexingUtil { public class ReindexingUtil {
public static final String ENTITY_TYPE_KEY = "entityType"; public static final String ENTITY_TYPE_KEY = "entityType";
@ -57,9 +57,9 @@ public class ReindexingUtil {
return success; return success;
} }
public static int getSuccessFromBulkResponseEs(org.elasticsearch.action.bulk.BulkResponse response) { public static int getSuccessFromBulkResponseEs(es.org.elasticsearch.action.bulk.BulkResponse response) {
int success = 0; int success = 0;
for (org.elasticsearch.action.bulk.BulkItemResponse bulkItemResponse : response) { for (es.org.elasticsearch.action.bulk.BulkItemResponse bulkItemResponse : response) {
if (!bulkItemResponse.isFailed()) { if (!bulkItemResponse.isFailed()) {
success++; success++;
} }

30
openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/SearchIndexWorkflow.java
View File

@ -20,6 +20,9 @@ import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getT
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.isDataInsightIndex; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.isDataInsightIndex;
import es.org.elasticsearch.action.bulk.BulkItemResponse;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.bulk.BulkResponse;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -28,9 +31,6 @@ import lombok.Getter;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang.exception.ExceptionUtils;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.analytics.ReportData; import org.openmetadata.schema.analytics.ReportData;
@ -151,11 +151,11 @@ public class SearchIndexWorkflow implements Runnable {
if (!resultList.getData().isEmpty()) { if (!resultList.getData().isEmpty()) {
if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) { if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
// process data to build Reindex Request // process data to build Reindex Request
org.opensearch.action.bulk.BulkRequest requests = os.org.opensearch.action.bulk.BulkRequest requests =
(org.opensearch.action.bulk.BulkRequest) entityProcessor.process(resultList, contextData); (os.org.opensearch.action.bulk.BulkRequest) entityProcessor.process(resultList, contextData);
// process data to build Reindex Request // process data to build Reindex Request
org.opensearch.action.bulk.BulkResponse response = os.org.opensearch.action.bulk.BulkResponse response =
(org.opensearch.action.bulk.BulkResponse) searchIndexSink.write(requests, contextData); (os.org.opensearch.action.bulk.BulkResponse) searchIndexSink.write(requests, contextData);
// update Status // update Status
handleErrorsOs(resultList, paginatedEntitiesSource.getLastFailedCursor(), response, currentTime); handleErrorsOs(resultList, paginatedEntitiesSource.getLastFailedCursor(), response, currentTime);
// Update stats // Update stats
@ -222,11 +222,11 @@ public class SearchIndexWorkflow implements Runnable {
if (!resultList.getData().isEmpty()) { if (!resultList.getData().isEmpty()) {
if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) { if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
// process data to build Reindex Request // process data to build Reindex Request
org.opensearch.action.bulk.BulkRequest requests = os.org.opensearch.action.bulk.BulkRequest requests =
(org.opensearch.action.bulk.BulkRequest) dataInsightProcessor.process(resultList, contextData); (os.org.opensearch.action.bulk.BulkRequest) dataInsightProcessor.process(resultList, contextData);
// process data to build Reindex Request // process data to build Reindex Request
org.opensearch.action.bulk.BulkResponse response = os.org.opensearch.action.bulk.BulkResponse response =
(org.opensearch.action.bulk.BulkResponse) searchIndexSink.write(requests, contextData); (os.org.opensearch.action.bulk.BulkResponse) searchIndexSink.write(requests, contextData);
handleErrorsOs(resultList, "", response, currentTime); handleErrorsOs(resultList, "", response, currentTime);
// Update stats // Update stats
success = searchRepository.getSearchClient().getSuccessFromBulkResponse(response); success = searchRepository.getSearchClient().getSuccessFromBulkResponse(response);
@ -338,7 +338,7 @@ public class SearchIndexWorkflow implements Runnable {
} }
private void handleErrorsOs( private void handleErrorsOs(
ResultList<?> data, String lastCursor, org.opensearch.action.bulk.BulkResponse response, long time) { ResultList<?> data, String lastCursor, os.org.opensearch.action.bulk.BulkResponse response, long time) {
handleSourceError(data, lastCursor, time); handleSourceError(data, lastCursor, time);
handleOsSinkErrors(response, time); handleOsSinkErrors(response, time);
} }
@ -388,11 +388,11 @@ public class SearchIndexWorkflow implements Runnable {
} }
@SneakyThrows @SneakyThrows
private void handleOsSinkErrors(org.opensearch.action.bulk.BulkResponse response, long time) { private void handleOsSinkErrors(os.org.opensearch.action.bulk.BulkResponse response, long time) {
List<FailureDetails> details = new ArrayList<>(); List<FailureDetails> details = new ArrayList<>();
for (org.opensearch.action.bulk.BulkItemResponse bulkItemResponse : response) { for (os.org.opensearch.action.bulk.BulkItemResponse bulkItemResponse : response) {
if (bulkItemResponse.isFailed()) { if (bulkItemResponse.isFailed()) {
org.opensearch.action.bulk.BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); os.org.opensearch.action.bulk.BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
FailureDetails esFailure = FailureDetails esFailure =
new FailureDetails() new FailureDetails()
.withContext( .withContext(

10
openmetadata-service/src/main/resources/elasticsearch/indexMapping.json
View File

@ -131,31 +131,31 @@
"alias": "team", "alias": "team",
"parentAliases": [] "parentAliases": []
}, },
"EntityReportData": { "entityReportData": {
"indexName": "entity_report_data_index", "indexName": "entity_report_data_index",
"indexMappingFile": "/elasticsearch/entity_report_data_index.json", "indexMappingFile": "/elasticsearch/entity_report_data_index.json",
"alias": "entityReportData", "alias": "entityReportData",
"parentAliases": [] "parentAliases": []
}, },
"WebAnalyticEntityViewReportData": { "webAnalyticEntityViewReportData": {
"indexName": "web_analytic_entity_view_report_data_index", "indexName": "web_analytic_entity_view_report_data_index",
"indexMappingFile": "/elasticsearch/web_analytic_entity_view_report_data_index.json", "indexMappingFile": "/elasticsearch/web_analytic_entity_view_report_data_index.json",
"alias": "webAnalyticEntityViewReportData", "alias": "webAnalyticEntityViewReportData",
"parentAliases": [] "parentAliases": []
}, },
"WebAnalyticUserActivityReportData": { "webAnalyticUserActivityReportData": {
"indexName": "web_analytic_user_activity_report_data_index", "indexName": "web_analytic_user_activity_report_data_index",
"indexMappingFile": "/elasticsearch/web_analytic_user_activity_report_data_index.json", "indexMappingFile": "/elasticsearch/web_analytic_user_activity_report_data_index.json",
"alias": "webAnalyticUserActivityReportData", "alias": "webAnalyticUserActivityReportData",
"parentAliases": [] "parentAliases": []
}, },
"RawCostAnalysisReportData": { "rawCostAnalysisReportData": {
"indexName": "raw_cost_analysis_report_data_index", "indexName": "raw_cost_analysis_report_data_index",
"indexMappingFile": "/elasticsearch/raw_cost_analysis_report_data_index.json", "indexMappingFile": "/elasticsearch/raw_cost_analysis_report_data_index.json",
"alias": "RawCostAnalysisReportData", "alias": "RawCostAnalysisReportData",
"parentAliases": [] "parentAliases": []
}, },
"AggregatedCostAnalysisReportData": { "aggregatedCostAnalysisReportData": {
"indexName": "aggregated_cost_analysis_report_data_index", "indexName": "aggregated_cost_analysis_report_data_index",
"indexMappingFile": "/elasticsearch/aggregated_cost_analysis_report_data_index.json", "indexMappingFile": "/elasticsearch/aggregated_cost_analysis_report_data_index.json",
"alias": "AggregatedCostAnalysisReportData", "alias": "AggregatedCostAnalysisReportData",

2
openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java
View File

@ -16,6 +16,7 @@ package org.openmetadata.service;
import static java.lang.String.format; import static java.lang.String.format;
import static org.openmetadata.service.util.TablesInitializer.validateAndRunSystemDataMigrations; import static org.openmetadata.service.util.TablesInitializer.validateAndRunSystemDataMigrations;
import es.org.elasticsearch.client.RestClient;
import io.dropwizard.jersey.jackson.JacksonFeature; import io.dropwizard.jersey.jackson.JacksonFeature;
import io.dropwizard.testing.ConfigOverride; import io.dropwizard.testing.ConfigOverride;
import io.dropwizard.testing.ResourceHelpers; import io.dropwizard.testing.ResourceHelpers;
@ -27,7 +28,6 @@ import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget; import javax.ws.rs.client.WebTarget;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.flywaydb.core.Flyway; import org.flywaydb.core.Flyway;
import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.client.ClientProperties;

32
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -52,6 +52,22 @@ import static org.openmetadata.service.util.TestUtils.UpdateType.NO_CHANGE;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import es.org.elasticsearch.action.search.SearchResponse;
import es.org.elasticsearch.client.Request;
import es.org.elasticsearch.client.Response;
import es.org.elasticsearch.client.RestClient;
import es.org.elasticsearch.search.SearchHit;
import es.org.elasticsearch.search.aggregations.Aggregation;
import es.org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import es.org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import es.org.elasticsearch.search.aggregations.metrics.ParsedTopHits;
import es.org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
import es.org.elasticsearch.xcontent.ContextParser;
import es.org.elasticsearch.xcontent.DeprecationHandler;
import es.org.elasticsearch.xcontent.NamedXContentRegistry;
import es.org.elasticsearch.xcontent.ParseField;
import es.org.elasticsearch.xcontent.XContentParser;
import es.org.elasticsearch.xcontent.json.JsonXContent;
import java.io.IOException; import java.io.IOException;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.time.Duration; import java.time.Duration;
@ -80,22 +96,6 @@ import org.apache.commons.text.RandomStringGenerator.Builder;
import org.apache.http.client.HttpResponseException; import org.apache.http.client.HttpResponseException;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.awaitility.Awaitility; import org.awaitility.Awaitility;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.metrics.ParsedTopHits;
import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
import org.elasticsearch.xcontent.ContextParser;
import org.elasticsearch.xcontent.DeprecationHandler;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.ParseField;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;

6
openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/ReportDataResourceTest.java
View File

@ -13,6 +13,9 @@ import static org.openmetadata.service.util.TestUtils.TEST_USER_NAME;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import es.org.elasticsearch.client.Request;
import es.org.elasticsearch.client.Response;
import es.org.elasticsearch.client.RestClient;
import java.io.IOException; import java.io.IOException;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
@ -22,9 +25,6 @@ import java.util.Map;
import java.util.UUID; import java.util.UUID;
import javax.ws.rs.client.WebTarget; import javax.ws.rs.client.WebTarget;
import org.apache.http.client.HttpResponseException; import org.apache.http.client.HttpResponseException;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.api.parallel.ExecutionMode;

29
openmetadata-shaded-deps/elasticsearch-dep/.gitignore vendored Normal file
View File

@ -0,0 +1,29 @@
### IntelliJ IDEA ###
out/
!**/src/main/**/out/
!**/src/test/**/out/
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store

111
openmetadata-shaded-deps/elasticsearch-dep/pom.xml Normal file
View File

@ -0,0 +1,111 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>openmetadata-shaded-deps</artifactId>
<groupId>org.open-metadata</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elasticsearch-deps</artifactId>
<name>Shaded Deps for OpenMetadata Server</name>
<description>Shaded version of dependencies used only for internal OpenMetadata code.</description>
<dependencies>
<!-- All dependencies should be marked as optional to work around
https://issues.apache.org/jira/browse/MSHADE-206 which makes it so
the dependency reduced pom is not reflected in a multi-module build
correctly. -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
<configuration>
<keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<minimizeJar>false</minimizeJar>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<relocations combine.children="append">
<relocation>
<pattern>org.elasticsearch</pattern>
<shadedPattern>es.org.elasticsearch</shadedPattern>
</relocation>
<relocation>
<pattern>org.elasticsearch.client</pattern>
<shadedPattern>es.org.elasticsearch.client</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.org.apache.lucene</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.sf</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.dsa</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/*.rsa</exclude>
<exclude>META-INF/*.EC</exclude>
<exclude>META-INF/*.ec</exclude>
<exclude>META-INF/MSFTSIG.SF</exclude>
<exclude>META-INF/MSFTSIG.RSA</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
</excludes>
</filter>
</filters>
</configuration>
</plugin>
<plugin>
<!--
Purpose of build-helper-maven-plugin is to ensure that IntelliJ sees shaded classes.
-->
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.4.0</version>
<executions>
<execution>
<id>workaround-makeItVisibleOnIntellij</id>
<phase>none</phase>
<goals>
<goal>attach-artifact</goal>
</goals>
<configuration>
<artifacts>
<artifact>
<file>${project.build.directory}/${project.build.finalName}.jar</file>
<type>jar</type>
<classifier>optional</classifier>
</artifact>
</artifacts>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

29
openmetadata-shaded-deps/opensearch-dep/.gitignore vendored Normal file
View File

@ -0,0 +1,29 @@
### IntelliJ IDEA ###
out/
!**/src/main/**/out/
!**/src/test/**/out/
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store

111
openmetadata-shaded-deps/opensearch-dep/pom.xml Normal file
View File

@ -0,0 +1,111 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>openmetadata-shaded-deps</artifactId>
<groupId>org.open-metadata</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>opensearch-deps</artifactId>
<name>OpenSearch Shaded Deps for OpenMetadata Server</name>
<description>Shaded version of OpenSearch dependencies used only for internal OpenMetadata code.</description>
<dependencies>
<!-- All dependencies should be marked as optional to work around
https://issues.apache.org/jira/browse/MSHADE-206 which makes it so
the dependency reduced pom is not reflected in a multi-module build
correctly. -->
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-high-level-client</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
<configuration>
<keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<minimizeJar>false</minimizeJar>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<relocations combine.children="append">
<relocation>
<pattern>org.opensearch</pattern>
<shadedPattern>os.org.opensearch</shadedPattern>
</relocation>
<relocation>
<pattern>org.opensearch.client</pattern>
<shadedPattern>os.org.opensearch.client</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.org.apache.lucene</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.sf</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.dsa</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/*.rsa</exclude>
<exclude>META-INF/*.EC</exclude>
<exclude>META-INF/*.ec</exclude>
<exclude>META-INF/MSFTSIG.SF</exclude>
<exclude>META-INF/MSFTSIG.RSA</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
</excludes>
</filter>
</filters>
</configuration>
</plugin>
<plugin>
<!--
Purpose of build-helper-maven-plugin is to ensure that IntelliJ sees shaded classes.
-->
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.4.0</version>
<executions>
<execution>
<id>workaround-makeItVisibleOnIntellij</id>
<phase>none</phase>
<goals>
<goal>attach-artifact</goal>
</goals>
<configuration>
<artifacts>
<artifact>
<file>${project.build.directory}/${project.build.finalName}.jar</file>
<type>jar</type>
<classifier>optional</classifier>
</artifact>
</artifacts>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

19
openmetadata-shaded-deps/pom.xml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>platform</artifactId>
<groupId>org.open-metadata</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>openmetadata-shaded-deps</artifactId>
<name>Shaded Deps for OpenMetadata Server</name>
<description>Shaded version of dependencies used only for internal OpenMetadata code.</description>
<modules>
<module>elasticsearch-dep</module>
<module>opensearch-dep</module>
</modules>
<packaging>pom</packaging>
</project>

51
pom.xml
View File

@ -31,6 +31,7 @@
<modules> <modules>
<module>openmetadata-spec</module> <module>openmetadata-spec</module>
<module>common</module> <module>common</module>
<module>openmetadata-shaded-deps</module>
<module>openmetadata-service</module> <module>openmetadata-service</module>
<module>openmetadata-ui</module> <module>openmetadata-ui</module>
<module>openmetadata-dist</module> <module>openmetadata-dist</module>
@ -107,8 +108,8 @@
<!-- We need ElasticSearch client to be compatible with both ElasticSearch and AWS OpenSearch <!-- We need ElasticSearch client to be compatible with both ElasticSearch and AWS OpenSearch
This compatibility broken in 7.14, so lets keep this version pinned to 7.13.x This compatibility broken in 7.14, so lets keep this version pinned to 7.13.x
--> -->
<elasticsearch.version>7.16.3</elasticsearch.version> <elasticsearch.version>7.17.13</elasticsearch.version>
<opensearch.version>1.3.0</opensearch.version> <opensearch.version>2.5.0</opensearch.version>
<httpasyncclient.version>4.1.5</httpasyncclient.version> <httpasyncclient.version>4.1.5</httpasyncclient.version>
<openapiswagger.version>2.2.15</openapiswagger.version> <openapiswagger.version>2.2.15</openapiswagger.version>
<spring-security-kerberos-core.version>1.0.1.RELEASE</spring-security-kerberos-core.version> <spring-security-kerberos-core.version>1.0.1.RELEASE</spring-security-kerberos-core.version>
@ -286,49 +287,17 @@
<artifactId>gson</artifactId> <artifactId>gson</artifactId>
<version>${gson.version}</version> <version>${gson.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${elasticsearch.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-grouping</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-memory</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial-extras</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial3d</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-suggest</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency> <dependency>
<groupId>org.opensearch.client</groupId> <groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-high-level-client</artifactId> <artifactId>opensearch-rest-high-level-client</artifactId>
<version>${opensearch.version}</version> <version>${opensearch.version}</version>
<!-- <classifier>shaded</classifier>-->
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${elasticsearch.version}</version>
<!-- <classifier>shaded</classifier>-->
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>