yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-28 09:13:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade search versions (#13479)

Browse Source 
* working on maven shading

* update es in docker

* add maven shade for elasticsearch/opensearch dependencies

* add submodule for es/os dependencies

* upgrade docker elasticsearch to latest 8.10.2

* upgrade docker elasticsearch to latest 8.10.2

* Update Postgres dev to use OpenSearch

* Fix container service dependency

* revert conf change

---------

Co-authored-by: 07Himank <himank07mehta@gmail.com>
Co-authored-by: 07Himank <112613760+07Himank@users.noreply.github.com>
This commit is contained in:
Sriharsha Chintalapani 2023-10-08 11:55:34 -07:00 committed by GitHub
parent b06db047a4
commit 35c4e18663
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
57 changed files with 839 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
docker/development/docker-compose-postgres.yml
View File

@ -24,7 +24,7 @@ services:
restart: always
command: "--work_mem=10MB"
depends_on:
- elasticsearch
- opensearch
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: password
@ -42,12 +42,13 @@ services:
timeout: 10s
retries: 10
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
container_name: openmetadata_elasticsearch
opensearch:
image: opensearchproject/opensearch:latest
container_name: openmetadata_opensearch
environment:
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- plugins.security.disabled=true
networks:
- local_app_net
expose:
@ -57,7 +58,7 @@ services:
- "9200:9200"
- "9300:9300"
volumes:
- es-data:/usr/share/elasticsearch/data
- es-data:/usr/share/opensearch/data
execute-migrate-all:
build:
@ -113,7 +114,7 @@ services:
DB_PORT: ${DB_PORT:-5432}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- opensearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
@ -211,7 +212,7 @@ services:
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
depends_on:
elasticsearch:
opensearch:
condition: service_started
postgresql:
condition: service_healthy
@ -344,7 +345,7 @@ services:
- "8586:8586"
- "5005:5005"
depends_on:
elasticsearch:
ingestion:
condition: service_started
postgresql:
condition: service_healthy
@ -376,7 +377,7 @@ services:
command:
- "/opt/airflow/ingestion_dependency.sh"
depends_on:
elasticsearch:
opensearch:
condition: service_started
postgresql:
condition: service_healthy

3
docker/development/docker-compose.yml
View File

@ -42,11 +42,12 @@ services:
- ./docker-volume/db-data:/var/lib/mysql
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
container_name: openmetadata_elasticsearch
environment:
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- xpack.security.enabled=false
networks:
- local_app_net
expose:

3
docker/docker-compose-quickstart/docker-compose-postgres.yml
View File

@ -41,10 +41,11 @@ services:
elasticsearch:
container_name: openmetadata_elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
environment:
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- xpack.security.enabled=false
networks:
- app_net
ports:

3
docker/docker-compose-quickstart/docker-compose.yml
View File

@ -39,10 +39,11 @@ services:
elasticsearch:
container_name: openmetadata_elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
environment:
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- xpack.security.enabled=false
networks:
- app_net
ports:

208
openmetadata-service/pom.xml
View File

@ -39,6 +39,16 @@
<artifactId>common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.open-metadata</groupId>
<artifactId>elasticsearch-deps</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.open-metadata</groupId>
<artifactId>opensearch-deps</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
@ -129,57 +139,6 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-grouping</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-memory</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial-extras</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial3d</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-suggest</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
</exclusion>
<exclusion>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-high-level-client</artifactId>
<version>${opensearch.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId>
@ -769,6 +728,153 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.1</version> <!-- Use the latest version available -->
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactId>my-app-shaded</shadedArtifactId>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
<relocations>
<!-- Relocate Elasticsearch's Lucene -->
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-core</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-analyzers-common</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-backward-codecs</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-grouping</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-highlighter</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-join</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-memory</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-misc</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-queries</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-queryparser</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-sandbox</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-spatial3d</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.lucene-suggest</shadedPattern>
</relocation>
<!-- Relocate OpenSearch's Lucene -->
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-core</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-analysis-common</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-backward-codecs</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-grouping</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-highlighter</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-join</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-memory</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-misc</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-queries</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-queryparser</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-sandbox</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-spatial-extras</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-spatial3d</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.lucene-suggest</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>module-info.class</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<!-- add Main-Class to manifest file -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.openmetadata.service.OpenMetadataApplication</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

4
openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
View File

@ -16,6 +16,8 @@ package org.openmetadata.service.resources.search;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.search.SearchRepository.ELASTIC_SEARCH_EXTENSION;
import es.org.elasticsearch.action.search.SearchResponse;
import es.org.elasticsearch.search.suggest.Suggest;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
@ -40,8 +42,6 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.suggest.Suggest;
import org.openmetadata.schema.api.CreateEventPublisherJob;
import org.openmetadata.schema.system.EventPublisherJob;
import org.openmetadata.service.Entity;

12
openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
View File

@ -17,9 +17,9 @@ import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.exception.CustomExceptionMessage;
import org.openmetadata.service.search.models.IndexMapping;
import org.openmetadata.service.util.SSLUtil;
import org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.bulk.BulkResponse;
import org.opensearch.client.RequestOptions;
import os.org.opensearch.action.bulk.BulkRequest;
import os.org.opensearch.action.bulk.BulkResponse;
import os.org.opensearch.client.RequestOptions;
public interface SearchClient {
@ -110,8 +110,8 @@ public interface SearchClient {
throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
}
default org.elasticsearch.action.bulk.BulkResponse bulk(
org.elasticsearch.action.bulk.BulkRequest data, org.elasticsearch.client.RequestOptions options)
default es.org.elasticsearch.action.bulk.BulkResponse bulk(
es.org.elasticsearch.action.bulk.BulkRequest data, es.org.elasticsearch.client.RequestOptions options)
throws IOException {
throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
}
@ -120,7 +120,7 @@ public interface SearchClient {
throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
}
default int getSuccessFromBulkResponse(org.elasticsearch.action.bulk.BulkResponse response) {
default int getSuccessFromBulkResponse(es.org.elasticsearch.action.bulk.BulkResponse response) {
throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
}

180
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View File

@ -19,6 +19,73 @@ import static org.openmetadata.service.search.EntityBuilderConstant.PRE_TAG;
import static org.openmetadata.service.search.EntityBuilderConstant.UNIFIED;
import static org.openmetadata.service.search.UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH;
import es.org.elasticsearch.action.ActionListener;
import es.org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import es.org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import es.org.elasticsearch.action.bulk.BulkItemResponse;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.bulk.BulkResponse;
import es.org.elasticsearch.action.delete.DeleteRequest;
import es.org.elasticsearch.action.delete.DeleteResponse;
import es.org.elasticsearch.action.search.SearchResponse;
import es.org.elasticsearch.action.support.WriteRequest;
import es.org.elasticsearch.action.support.master.AcknowledgedResponse;
import es.org.elasticsearch.action.update.UpdateRequest;
import es.org.elasticsearch.action.update.UpdateResponse;
import es.org.elasticsearch.client.RequestOptions;
import es.org.elasticsearch.client.RestClient;
import es.org.elasticsearch.client.RestClientBuilder;
import es.org.elasticsearch.client.RestHighLevelClient;
import es.org.elasticsearch.client.RestHighLevelClientBuilder;
import es.org.elasticsearch.client.indices.CreateIndexRequest;
import es.org.elasticsearch.client.indices.CreateIndexResponse;
import es.org.elasticsearch.client.indices.GetIndexRequest;
import es.org.elasticsearch.client.indices.PutMappingRequest;
import es.org.elasticsearch.common.lucene.search.function.CombineFunction;
import es.org.elasticsearch.common.settings.Settings;
import es.org.elasticsearch.common.unit.Fuzziness;
import es.org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import es.org.elasticsearch.core.TimeValue;
import es.org.elasticsearch.index.query.BoolQueryBuilder;
import es.org.elasticsearch.index.query.MatchQueryBuilder;
import es.org.elasticsearch.index.query.MultiMatchQueryBuilder;
import es.org.elasticsearch.index.query.Operator;
import es.org.elasticsearch.index.query.QueryBuilder;
import es.org.elasticsearch.index.query.QueryBuilders;
import es.org.elasticsearch.index.query.QueryStringQueryBuilder;
import es.org.elasticsearch.index.query.RangeQueryBuilder;
import es.org.elasticsearch.index.query.ScriptQueryBuilder;
import es.org.elasticsearch.index.query.TermQueryBuilder;
import es.org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionBuilder;
import es.org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import es.org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import es.org.elasticsearch.index.reindex.BulkByScrollResponse;
import es.org.elasticsearch.index.reindex.DeleteByQueryRequest;
import es.org.elasticsearch.index.reindex.UpdateByQueryRequest;
import es.org.elasticsearch.script.Script;
import es.org.elasticsearch.script.ScriptType;
import es.org.elasticsearch.search.SearchModule;
import es.org.elasticsearch.search.aggregations.AggregationBuilder;
import es.org.elasticsearch.search.aggregations.AggregationBuilders;
import es.org.elasticsearch.search.aggregations.BucketOrder;
import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import es.org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
import es.org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import es.org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import es.org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import es.org.elasticsearch.search.builder.SearchSourceBuilder;
import es.org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import es.org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import es.org.elasticsearch.search.sort.SortOrder;
import es.org.elasticsearch.search.suggest.Suggest;
import es.org.elasticsearch.search.suggest.SuggestBuilder;
import es.org.elasticsearch.search.suggest.SuggestBuilders;
import es.org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import es.org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
import es.org.elasticsearch.xcontent.NamedXContentRegistry;
import es.org.elasticsearch.xcontent.XContentParser;
import es.org.elasticsearch.xcontent.XContentType;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
@ -41,72 +108,6 @@ import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.ScriptQueryBuilder;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.query.functionscore.FieldValueFactorFunctionBuilder;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestBuilders;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentType;
import org.openmetadata.schema.DataInsightInterface;
import org.openmetadata.schema.dataInsight.DataInsightChartResult;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
@ -211,18 +212,6 @@ public class ElasticSearchClient implements SearchClient {
@Override
public void createAliases(IndexMapping indexMapping) {
try {
ActionListener<AcknowledgedResponse> listener =
new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
LOG.debug("Created successfully: " + acknowledgedResponse.toString());
}
@Override
public void onFailure(Exception e) {
LOG.error("Creation failed: " + e.getMessage());
}
};
Set<String> aliases = new HashSet<>(indexMapping.getParentAliases());
aliases.add(indexMapping.getAlias());
IndicesAliasesRequest.AliasActions aliasAction =
@ -231,7 +220,7 @@ public class ElasticSearchClient implements SearchClient {
.aliases(aliases.toArray(new String[0]));
IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
aliasesRequest.addAliasAction(aliasAction);
client.indices().updateAliasesAsync(aliasesRequest, RequestOptions.DEFAULT, listener);
client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
} catch (Exception e) {
LOG.error(String.format("Failed to create alias for %s due to", indexMapping.getIndexName()), e);
}
@ -384,7 +373,7 @@ public class ElasticSearchClient implements SearchClient {
String response =
client
.search(
new org.elasticsearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder),
new es.org.elasticsearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder),
RequestOptions.DEFAULT)
.toString();
return Response.status(OK).entity(response).build();
@ -392,8 +381,8 @@ public class ElasticSearchClient implements SearchClient {
@Override
public Response searchBySourceUrl(String sourceUrl) throws IOException {
org.elasticsearch.action.search.SearchRequest searchRequest =
new org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS);
es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("sourceUrl", sourceUrl)));
searchRequest.source(searchSourceBuilder);
@ -403,8 +392,8 @@ public class ElasticSearchClient implements SearchClient {
@Override
public Response searchByField(String fieldName, String fieldValue, String index) throws IOException {
org.elasticsearch.action.search.SearchRequest searchRequest =
new org.elasticsearch.action.search.SearchRequest(index);
es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.wildcardQuery(fieldName, fieldValue));
searchRequest.source(searchSourceBuilder);
@ -433,7 +422,7 @@ public class ElasticSearchClient implements SearchClient {
String response =
client
.search(
new org.elasticsearch.action.search.SearchRequest(index).source(searchSourceBuilder),
new es.org.elasticsearch.action.search.SearchRequest(index).source(searchSourceBuilder),
RequestOptions.DEFAULT)
.toString();
return Response.status(OK).entity(response).build();
@ -463,8 +452,8 @@ public class ElasticSearchClient implements SearchClient {
.fetchSource(
new FetchSourceContext(
request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {}));
org.elasticsearch.action.search.SearchRequest searchRequest =
new org.elasticsearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder);
es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Suggest suggest = searchResponse.getSuggest();
return Response.status(OK).entity(suggest.toString()).build();
@ -1119,7 +1108,7 @@ public class ElasticSearchClient implements SearchClient {
DataInsightChartResult.DataInsightChartType chartType,
String indexName)
throws IOException, ParseException {
org.elasticsearch.action.search.SearchRequest searchRequestTotalAssets =
es.org.elasticsearch.action.search.SearchRequest searchRequestTotalAssets =
buildSearchRequest(scheduleTime, currentTime, null, team, chartType, null, null, null, indexName);
SearchResponse searchResponseTotalAssets = client.search(searchRequestTotalAssets, RequestOptions.DEFAULT);
DataInsightChartResult processedDataTotalAssets =
@ -1150,7 +1139,7 @@ public class ElasticSearchClient implements SearchClient {
String queryFilter,
String dataReportIndex)
throws IOException, ParseException {
org.elasticsearch.action.search.SearchRequest searchRequest =
es.org.elasticsearch.action.search.SearchRequest searchRequest =
buildSearchRequest(startTs, endTs, tier, team, dataInsightChartName, size, from, queryFilter, dataReportIndex);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return Response.status(OK).entity(processDataInsightChartResult(searchResponse, dataInsightChartName)).build();
@ -1203,7 +1192,7 @@ public class ElasticSearchClient implements SearchClient {
}
}
private static org.elasticsearch.action.search.SearchRequest buildSearchRequest(
private static es.org.elasticsearch.action.search.SearchRequest buildSearchRequest(
Long startTs,
Long endTs,
String tier,
@ -1229,8 +1218,8 @@ public class ElasticSearchClient implements SearchClient {
searchSourceBuilder.sort("data.lifeCycle.accessed.timestamp", SortOrder.DESC);
}
org.elasticsearch.action.search.SearchRequest searchRequest =
new org.elasticsearch.action.search.SearchRequest(dataReportIndex);
es.org.elasticsearch.action.search.SearchRequest searchRequest =
new es.org.elasticsearch.action.search.SearchRequest(dataReportIndex);
searchRequest.source(searchSourceBuilder);
return searchRequest;
}
@ -1478,6 +1467,10 @@ public class ElasticSearchClient implements SearchClient {
try {
RestClientBuilder restClientBuilder =
RestClient.builder(new HttpHost(esConfig.getHost(), esConfig.getPort(), esConfig.getScheme()));
RestClient restClient =
RestClient.builder(new HttpHost(esConfig.getHost(), esConfig.getPort(), esConfig.getScheme())).build();
if (StringUtils.isNotEmpty(esConfig.getUsername()) && StringUtils.isNotEmpty(esConfig.getPassword())) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
@ -1502,7 +1495,8 @@ public class ElasticSearchClient implements SearchClient {
requestConfigBuilder
.setConnectTimeout(esConfig.getConnectionTimeoutSecs() * 1000)
.setSocketTimeout(esConfig.getSocketTimeoutSecs() * 1000));
return new RestHighLevelClient(restClientBuilder);
// return new RestHighLevelClient(restClientBuilder);
return new RestHighLevelClientBuilder(restClient).setApiCompatibilityMode(true).build();
} catch (Exception e) {
LOG.error("Failed to create elastic search client ", e);
return null;

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java
View File

@ -3,12 +3,12 @@ package org.openmetadata.service.search.elasticsearch;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.update.UpdateRequest;
import es.org.elasticsearch.xcontent.XContentType;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.xcontent.XContentType;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.analytics.ReportData;
import org.openmetadata.schema.system.StepStats;

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java
View File

@ -3,13 +3,13 @@ package org.openmetadata.service.search.elasticsearch;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.update.UpdateRequest;
import es.org.elasticsearch.xcontent.XContentType;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.xcontent.XContentType;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.system.StepStats;

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexSink.java
View File

@ -3,11 +3,11 @@ package org.openmetadata.service.search.elasticsearch;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getSuccessFromBulkResponseEs;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.bulk.BulkResponse;
import es.org.elasticsearch.client.RequestOptions;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.RequestOptions;
import org.openmetadata.schema.system.StepStats;
import org.openmetadata.service.exception.SinkException;
import org.openmetadata.service.search.SearchRepository;

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsCountAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsCountAggregator;
public class ElasticSearchAggregatedUnusedAssetsCountAggregator

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUnusedAssetsSizeAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsSizeAggregator;
public class ElasticSearchAggregatedUnusedAssetsSizeAggregator

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsCountAggregator;
public class ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsSizeAggregator;
public class ElasticSearchAggregatedUsedvsUnusedAssetsSizeAggregator

4
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchDailyActiveUsersAggregator.java
View File

@ -1,8 +1,8 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.openmetadata.service.dataInsight.DailyActiveUsersAggregator;
public class ElasticSearchDailyActiveUsersAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchEntitiesDescriptionAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.EntitiesDescriptionAggregator;
public class ElasticSearchEntitiesDescriptionAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchEntitiesOwnerAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.EntitiesOwnerAggregator;
public class ElasticSearchEntitiesOwnerAggregator

8
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchMostActiveUsersAggregator.java
View File

@ -1,10 +1,10 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Max;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Max;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.MostActiveUsersAggregator;
public class ElasticSearchMostActiveUsersAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchMostViewedEntitiesAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.MostViewedEntitiesAggregator;
public class ElasticSearchMostViewedEntitiesAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchPageViewsByEntitiesAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.PageViewsByEntitiesAggregator;
public class ElasticSearchPageViewsByEntitiesAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchServicesDescriptionAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.ServicesDescriptionAggregator;
public class ElasticSearchServicesDescriptionAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchServicesOwnerAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.ServicesOwnerAggregator;
public class ElasticSearchServicesOwnerAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchTotalEntitiesAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.TotalEntitiesAggregator;
public class ElasticSearchTotalEntitiesAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchTotalEntitiesByTierAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import es.org.elasticsearch.search.aggregations.Aggregations;
import es.org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import es.org.elasticsearch.search.aggregations.metrics.Sum;
import java.util.List;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.openmetadata.service.dataInsight.TotalEntitiesByTierAggregator;
public class ElasticSearchTotalEntitiesByTierAggregator

6
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/dataInsightAggregators/ElasticSearchUnusedAssetsAggregator.java
View File

@ -1,8 +1,8 @@
package org.openmetadata.service.search.elasticsearch.dataInsightAggregators;
import org.apache.lucene.search.TotalHits;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import es.org.apache.lucene.search.TotalHits;
import es.org.elasticsearch.search.SearchHit;
import es.org.elasticsearch.search.SearchHits;
import org.openmetadata.service.dataInsight.UnusedAssetsAggregator;
public class ElasticSearchUnusedAssetsAggregator extends UnusedAssetsAggregator<SearchHits, SearchHit, TotalHits> {

216
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View File

@ -47,7 +47,6 @@ import org.openmetadata.service.dataInsight.DataInsightAggregatorInterface;
import org.openmetadata.service.jdbi3.DataInsightChartRepository;
import org.openmetadata.service.search.SearchClient;
import org.openmetadata.service.search.SearchRequest;
import org.openmetadata.service.search.UpdateSearchEventsConstant;
import org.openmetadata.service.search.indexes.ContainerIndex;
import org.openmetadata.service.search.indexes.DashboardDataModelIndex;
import org.openmetadata.service.search.indexes.DashboardIndex;
@ -80,72 +79,72 @@ import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSear
import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchTotalEntitiesByTierAggregator;
import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchUnusedAssetsAggregator;
import org.openmetadata.service.util.JsonUtils;
import org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
import org.opensearch.action.bulk.BulkItemResponse;
import org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.bulk.BulkResponse;
import org.opensearch.action.delete.DeleteRequest;
import org.opensearch.action.delete.DeleteResponse;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.action.support.WriteRequest;
import org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.action.update.UpdateRequest;
import org.opensearch.action.update.UpdateResponse;
import org.opensearch.client.RequestOptions;
import org.opensearch.client.RestClient;
import org.opensearch.client.RestClientBuilder;
import org.opensearch.client.RestHighLevelClient;
import org.opensearch.client.indices.CreateIndexRequest;
import org.opensearch.client.indices.CreateIndexResponse;
import org.opensearch.client.indices.GetIndexRequest;
import org.opensearch.client.indices.PutMappingRequest;
import org.opensearch.common.lucene.search.function.CombineFunction;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.Fuzziness;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.NamedXContentRegistry;
import org.opensearch.common.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.MatchQueryBuilder;
import org.opensearch.index.query.MultiMatchQueryBuilder;
import org.opensearch.index.query.Operator;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.QueryStringQueryBuilder;
import org.opensearch.index.query.RangeQueryBuilder;
import org.opensearch.index.query.ScriptQueryBuilder;
import org.opensearch.index.query.TermQueryBuilder;
import org.opensearch.index.query.functionscore.FieldValueFactorFunctionBuilder;
import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.opensearch.index.query.functionscore.ScoreFunctionBuilders;
import org.opensearch.index.reindex.BulkByScrollResponse;
import org.opensearch.index.reindex.DeleteByQueryRequest;
import org.opensearch.index.reindex.UpdateByQueryRequest;
import org.opensearch.script.Script;
import org.opensearch.script.ScriptType;
import org.opensearch.search.SearchModule;
import org.opensearch.search.aggregations.AggregationBuilder;
import org.opensearch.search.aggregations.AggregationBuilders;
import org.opensearch.search.aggregations.BucketOrder;
import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.opensearch.search.aggregations.bucket.terms.IncludeExclude;
import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.opensearch.search.aggregations.metrics.SumAggregationBuilder;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.search.fetch.subphase.FetchSourceContext;
import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.opensearch.search.sort.SortOrder;
import org.opensearch.search.suggest.Suggest;
import org.opensearch.search.suggest.SuggestBuilder;
import org.opensearch.search.suggest.SuggestBuilders;
import org.opensearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.opensearch.search.suggest.completion.context.CategoryQueryContext;
import os.org.opensearch.action.ActionListener;
import os.org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;
import os.org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
import os.org.opensearch.action.bulk.BulkItemResponse;
import os.org.opensearch.action.bulk.BulkRequest;
import os.org.opensearch.action.bulk.BulkResponse;
import os.org.opensearch.action.delete.DeleteRequest;
import os.org.opensearch.action.delete.DeleteResponse;
import os.org.opensearch.action.search.SearchResponse;
import os.org.opensearch.action.support.WriteRequest;
import os.org.opensearch.action.support.master.AcknowledgedResponse;
import os.org.opensearch.action.update.UpdateRequest;
import os.org.opensearch.action.update.UpdateResponse;
import os.org.opensearch.client.RequestOptions;
import os.org.opensearch.client.RestClient;
import os.org.opensearch.client.RestClientBuilder;
import os.org.opensearch.client.RestHighLevelClient;
import os.org.opensearch.client.indices.CreateIndexRequest;
import os.org.opensearch.client.indices.CreateIndexResponse;
import os.org.opensearch.client.indices.GetIndexRequest;
import os.org.opensearch.client.indices.PutMappingRequest;
import os.org.opensearch.common.lucene.search.function.CombineFunction;
import os.org.opensearch.common.settings.Settings;
import os.org.opensearch.common.unit.Fuzziness;
import os.org.opensearch.common.unit.TimeValue;
import os.org.opensearch.common.xcontent.LoggingDeprecationHandler;
import os.org.opensearch.common.xcontent.NamedXContentRegistry;
import os.org.opensearch.common.xcontent.XContentParser;
import os.org.opensearch.common.xcontent.XContentType;
import os.org.opensearch.index.query.BoolQueryBuilder;
import os.org.opensearch.index.query.MatchQueryBuilder;
import os.org.opensearch.index.query.MultiMatchQueryBuilder;
import os.org.opensearch.index.query.Operator;
import os.org.opensearch.index.query.QueryBuilder;
import os.org.opensearch.index.query.QueryBuilders;
import os.org.opensearch.index.query.QueryStringQueryBuilder;
import os.org.opensearch.index.query.RangeQueryBuilder;
import os.org.opensearch.index.query.ScriptQueryBuilder;
import os.org.opensearch.index.query.TermQueryBuilder;
import os.org.opensearch.index.query.functionscore.FieldValueFactorFunctionBuilder;
import os.org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder;
import os.org.opensearch.index.query.functionscore.ScoreFunctionBuilders;
import os.org.opensearch.index.reindex.BulkByScrollResponse;
import os.org.opensearch.index.reindex.DeleteByQueryRequest;
import os.org.opensearch.index.reindex.UpdateByQueryRequest;
import os.org.opensearch.script.Script;
import os.org.opensearch.script.ScriptType;
import os.org.opensearch.search.SearchModule;
import os.org.opensearch.search.aggregations.AggregationBuilder;
import os.org.opensearch.search.aggregations.AggregationBuilders;
import os.org.opensearch.search.aggregations.BucketOrder;
import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import os.org.opensearch.search.aggregations.bucket.terms.IncludeExclude;
import os.org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import os.org.opensearch.search.aggregations.metrics.MaxAggregationBuilder;
import os.org.opensearch.search.aggregations.metrics.SumAggregationBuilder;
import os.org.opensearch.search.builder.SearchSourceBuilder;
import os.org.opensearch.search.fetch.subphase.FetchSourceContext;
import os.org.opensearch.search.fetch.subphase.highlight.HighlightBuilder;
import os.org.opensearch.search.sort.SortOrder;
import os.org.opensearch.search.suggest.Suggest;
import os.org.opensearch.search.suggest.SuggestBuilder;
import os.org.opensearch.search.suggest.SuggestBuilders;
import os.org.opensearch.search.suggest.completion.CompletionSuggestionBuilder;
import os.org.opensearch.search.suggest.completion.context.CategoryQueryContext;
@Slf4j
// Not tagged with Repository annotation as it is programmatically initialized
@ -155,7 +154,7 @@ public class OpenSearchClient implements SearchClient {
private final boolean isClientAvailable;
static {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, List.of());
SearchModule searchModule = new SearchModule(Settings.EMPTY, List.of());
X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents());
}
@ -192,13 +191,13 @@ public class OpenSearchClient implements SearchClient {
// creating alias for indexes
createAliases(indexMapping);
} catch (Exception e) {
LOG.error("Failed to create Elastic Search indexes due to", e);
LOG.error("Failed to create Open Search indexes due to", e);
return false;
}
return true;
} else {
LOG.error(
"Failed to create Elastic Search index as client is not property configured, Please check your OpenMetadata configuration");
"Failed to create Open Search index as client is not property configured, Please check your OpenMetadata configuration");
return false;
}
}
@ -206,27 +205,15 @@ public class OpenSearchClient implements SearchClient {
@Override
public void createAliases(IndexMapping indexMapping) {
try {
ActionListener<AcknowledgedResponse> listener =
new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
LOG.debug("Created successfully: " + acknowledgedResponse.toString());
}
@Override
public void onFailure(Exception e) {
LOG.error("Creation failed: " + e.getMessage());
}
};
List<String> aliases = indexMapping.getParentAliases();
aliases.add(indexMapping.getAlias());
IndicesAliasesRequest.AliasActions aliasAction =
IndicesAliasesRequest.AliasActions.add()
.index(indexMapping.getIndexMappingFile())
.index(indexMapping.getIndexName())
.aliases(aliases.toArray(new String[0]));
IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
aliasesRequest.addAliasAction(aliasAction);
client.indices().updateAliasesAsync(aliasesRequest, RequestOptions.DEFAULT, listener);
client.indices().updateAliases(aliasesRequest, RequestOptions.DEFAULT);
} catch (Exception e) {
LOG.error(String.format("Failed to create alias for %s due to", indexMapping.getIndexName()), e);
}
@ -240,7 +227,7 @@ public class OpenSearchClient implements SearchClient {
AcknowledgedResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT);
LOG.debug("{} Updated {}", indexMapping.getIndexMappingFile(), putMappingResponse.isAcknowledged());
} catch (Exception e) {
LOG.error(String.format("Failed to Update Elastic Search index %s due to", indexMapping.getIndexName()), e);
LOG.error(String.format("Failed to Update Open Search index %s due to", indexMapping.getIndexName()), e);
}
}
@ -251,7 +238,7 @@ public class OpenSearchClient implements SearchClient {
AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
LOG.debug("{} Deleted {}", indexMapping.getIndexName(), deleteIndexResponse.isAcknowledged());
} catch (IOException e) {
LOG.error("Failed to delete Elastic Search indexes due to", e);
LOG.error("Failed to delete Open Search indexes due to", e);
}
}
@ -359,11 +346,11 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder.sort(request.getSortFieldParam(), SortOrder.fromString(request.getSortOrder()));
}
/* for performance reasons ElasticSearch doesn't provide accurate hits
/* for performance reasons OpenSearch doesn't provide accurate hits
if we enable trackTotalHits parameter it will try to match every result, count and return hits
however in most cases for search results an approximate value is good enough.
we are displaying total entity counts in landing page and explore page where we need the total count
https://github.com/elastic/elasticsearch/issues/33028 */
https://github.com/Open/Opensearch/issues/33028 */
searchSourceBuilder.fetchSource(
new FetchSourceContext(
request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {}));
@ -378,7 +365,7 @@ public class OpenSearchClient implements SearchClient {
String response =
client
.search(
new org.opensearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder),
new os.org.opensearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder),
RequestOptions.DEFAULT)
.toString();
return Response.status(OK).entity(response).build();
@ -387,8 +374,8 @@ public class OpenSearchClient implements SearchClient {
@Override
public Response searchBySourceUrl(String sourceUrl) throws IOException {
QueryBuilder wildcardQuery = QueryBuilders.queryStringQuery(sourceUrl).field("sourceUrl").escape(true);
org.opensearch.action.search.SearchRequest searchRequest =
new org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS);
os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(GLOBAL_SEARCH_ALIAS);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(wildcardQuery);
searchRequest.source(searchSourceBuilder);
@ -398,7 +385,8 @@ public class OpenSearchClient implements SearchClient {
@Override
public Response searchByField(String fieldName, String fieldValue, String index) throws IOException {
org.opensearch.action.search.SearchRequest searchRequest = new org.opensearch.action.search.SearchRequest(index);
os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.wildcardQuery(fieldName, fieldValue));
searchRequest.source(searchSourceBuilder);
@ -426,7 +414,7 @@ public class OpenSearchClient implements SearchClient {
String response =
client
.search(
new org.opensearch.action.search.SearchRequest(index).source(searchSourceBuilder),
new os.org.opensearch.action.search.SearchRequest(index).source(searchSourceBuilder),
RequestOptions.DEFAULT)
.toString();
return Response.status(OK).entity(response).build();
@ -441,12 +429,12 @@ public class OpenSearchClient implements SearchClient {
new ActionListener<>() {
@Override
public void onResponse(UpdateResponse updateResponse) {
LOG.debug("Created successfully: " + updateResponse.toString());
LOG.debug("Entity Updated successfully: " + updateResponse.toString());
}
@Override
public void onFailure(Exception e) {
LOG.error("Creation failed: " + e.getMessage());
LOG.error("Entity Update failed: " + e.getMessage());
}
};
client.updateAsync(updateRequest, RequestOptions.DEFAULT, listener);
@ -476,8 +464,8 @@ public class OpenSearchClient implements SearchClient {
.fetchSource(
new FetchSourceContext(
request.fetchSource(), request.getIncludeSourceFields().toArray(String[]::new), new String[] {}));
org.opensearch.action.search.SearchRequest searchRequest =
new org.opensearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder);
os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(request.getIndex()).source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Suggest suggest = searchResponse.getSuggest();
return Response.status(OK).entity(suggest.toString()).build();
@ -932,7 +920,7 @@ public class OpenSearchClient implements SearchClient {
ScriptQueryBuilder scriptQuery = new ScriptQueryBuilder(script);
DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexName);
deleteByQueryRequest.setQuery(scriptQuery);
deleteEntityFromElasticSearchByQuery(deleteByQueryRequest);
deleteEntityFromOpenSearchByQuery(deleteByQueryRequest);
}
}
@ -940,7 +928,7 @@ public class OpenSearchClient implements SearchClient {
public void deleteEntity(String indexName, String docId) {
if (isClientAvailable) {
DeleteRequest deleteRequest = new DeleteRequest(indexName, docId);
deleteEntityFromElasticSearch(deleteRequest);
deleteEntityFromOpenSearch(deleteRequest);
}
}
@ -953,7 +941,7 @@ public class OpenSearchClient implements SearchClient {
queryBuilder.must(new TermQueryBuilder(p.getKey(), p.getValue()));
}
deleteByQueryRequest.setQuery(queryBuilder);
deleteEntityFromElasticSearchByQuery(deleteByQueryRequest);
deleteEntityFromOpenSearchByQuery(deleteByQueryRequest);
}
}
@ -979,7 +967,7 @@ public class OpenSearchClient implements SearchClient {
updateByQueryRequest.setQuery(queryBuilder);
Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptTxt, new HashMap<>());
updateByQueryRequest.setScript(script);
updateElasticSearchByQuery(updateByQueryRequest);
updateOpenSearchByQuery(updateByQueryRequest);
}
}
@ -1007,11 +995,11 @@ public class OpenSearchClient implements SearchClient {
updates.getKey(),
JsonUtils.getMap(updates.getValue() == null ? new HashMap<>() : updates.getValue()));
updateByQueryRequest.setScript(script);
updateElasticSearchByQuery(updateByQueryRequest);
updateOpenSearchByQuery(updateByQueryRequest);
}
}
private void updateElasticSearchByQuery(UpdateByQueryRequest updateByQueryRequest) {
private void updateOpenSearchByQuery(UpdateByQueryRequest updateByQueryRequest) {
if (updateByQueryRequest != null && isClientAvailable) {
updateByQueryRequest.setRefresh(true);
LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest);
@ -1034,26 +1022,26 @@ public class OpenSearchClient implements SearchClient {
public void updateOpenSearch(UpdateRequest updateRequest) {
if (updateRequest != null && isClientAvailable) {
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
LOG.debug(UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH, updateRequest);
LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateRequest);
ActionListener<UpdateResponse> listener =
new ActionListener<>() {
@Override
public void onResponse(UpdateResponse updateResponse) {
LOG.debug("Created successfully: " + updateResponse.toString());
LOG.debug("Entity Updated successfully: " + updateResponse.toString());
}
@Override
public void onFailure(Exception e) {
LOG.error("Creation failed: " + e.getMessage());
LOG.error("Entity Update failed: " + e.getMessage());
}
};
client.updateAsync(updateRequest, RequestOptions.DEFAULT, listener);
}
}
private void deleteEntityFromElasticSearch(DeleteRequest deleteRequest) {
private void deleteEntityFromOpenSearch(DeleteRequest deleteRequest) {
if (deleteRequest != null && isClientAvailable) {
LOG.debug(UpdateSearchEventsConstant.SENDING_REQUEST_TO_ELASTIC_SEARCH, deleteRequest);
LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, deleteRequest);
ActionListener<DeleteResponse> listener =
new ActionListener<>() {
@Override
@ -1071,7 +1059,7 @@ public class OpenSearchClient implements SearchClient {
}
}
private void deleteEntityFromElasticSearchByQuery(DeleteByQueryRequest deleteRequest) {
private void deleteEntityFromOpenSearchByQuery(DeleteByQueryRequest deleteRequest) {
if (deleteRequest != null && isClientAvailable) {
deleteRequest.setRefresh(true);
ActionListener<BulkByScrollResponse> listener =
@ -1124,7 +1112,7 @@ public class OpenSearchClient implements SearchClient {
DataInsightChartResult.DataInsightChartType chartType,
String indexName)
throws IOException, ParseException {
org.opensearch.action.search.SearchRequest searchRequestTotalAssets =
os.org.opensearch.action.search.SearchRequest searchRequestTotalAssets =
buildSearchRequest(scheduleTime, currentTime, null, team, chartType, null, null, null, indexName);
SearchResponse searchResponseTotalAssets = client.search(searchRequestTotalAssets, RequestOptions.DEFAULT);
DataInsightChartResult processedDataTotalAssets =
@ -1155,7 +1143,7 @@ public class OpenSearchClient implements SearchClient {
String queryFilter,
String dataReportIndex)
throws IOException, ParseException {
org.opensearch.action.search.SearchRequest searchRequest =
os.org.opensearch.action.search.SearchRequest searchRequest =
buildSearchRequest(startTs, endTs, tier, team, dataInsightChartName, size, from, queryFilter, dataReportIndex);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return Response.status(OK).entity(processDataInsightChartResult(searchResponse, dataInsightChartName)).build();
@ -1208,7 +1196,7 @@ public class OpenSearchClient implements SearchClient {
}
}
private static org.opensearch.action.search.SearchRequest buildSearchRequest(
private static os.org.opensearch.action.search.SearchRequest buildSearchRequest(
Long startTs,
Long endTs,
String tier,
@ -1233,8 +1221,8 @@ public class OpenSearchClient implements SearchClient {
searchSourceBuilder.sort("data.lifeCycle.accessed.timestamp", SortOrder.DESC);
}
org.opensearch.action.search.SearchRequest searchRequest =
new org.opensearch.action.search.SearchRequest(dataReportIndex);
os.org.opensearch.action.search.SearchRequest searchRequest =
new os.org.opensearch.action.search.SearchRequest(dataReportIndex);
searchRequest.source(searchSourceBuilder);
return searchRequest;
}

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java
View File

@ -16,9 +16,9 @@ import org.openmetadata.service.search.models.IndexMapping;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.workflows.interfaces.Processor;
import org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.update.UpdateRequest;
import org.opensearch.common.xcontent.XContentType;
import os.org.opensearch.action.bulk.BulkRequest;
import os.org.opensearch.action.update.UpdateRequest;
import os.org.opensearch.common.xcontent.XContentType;
@Slf4j
public class OpenSearchDataInsightProcessor implements Processor<BulkRequest, ResultList<ReportData>> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java
View File

@ -17,9 +17,9 @@ import org.openmetadata.service.search.models.IndexMapping;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.workflows.interfaces.Processor;
import org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.update.UpdateRequest;
import org.opensearch.common.xcontent.XContentType;
import os.org.opensearch.action.bulk.BulkRequest;
import os.org.opensearch.action.update.UpdateRequest;
import os.org.opensearch.common.xcontent.XContentType;
@Slf4j
public class OpenSearchEntitiesProcessor implements Processor<BulkRequest, ResultList<? extends EntityInterface>> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexSink.java
View File

@ -9,9 +9,9 @@ import org.openmetadata.schema.system.StepStats;
import org.openmetadata.service.exception.SinkException;
import org.openmetadata.service.search.SearchRepository;
import org.openmetadata.service.workflows.interfaces.Sink;
import org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.bulk.BulkResponse;
import org.opensearch.client.RequestOptions;
import os.org.opensearch.action.bulk.BulkRequest;
import os.org.opensearch.action.bulk.BulkResponse;
import os.org.opensearch.client.RequestOptions;
@Slf4j
public class OpenSearchIndexSink implements Sink<BulkRequest, BulkResponse> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsCountAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsCountAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchAggregatedUnusedAssetsCountAggregator
extends AggregatedUnusedAssetsCountAggregator<Aggregations, Histogram, Bucket, Sum> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUnusedAssetsSizeAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.AggregatedUnusedAssetsSizeAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchAggregatedUnusedAssetsSizeAggregator
extends AggregatedUnusedAssetsSizeAggregator<Aggregations, Histogram, Bucket, Sum> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsCountAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator
extends AggregatedUsedvsUnusedAssetsCountAggregator<Aggregations, Histogram, Bucket, Sum> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.AggregatedUsedvsUnusedAssetsSizeAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchAggregatedUsedvsUnusedAssetsSizeAggregator
extends AggregatedUsedvsUnusedAssetsSizeAggregator<Aggregations, Histogram, Bucket, Sum> {

4
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchDailyActiveUsersAggregator.java
View File

@ -1,8 +1,8 @@
package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.histogram.Histogram;
public class OpenSearchDailyActiveUsersAggregator
extends org.openmetadata.service.dataInsight.DailyActiveUsersAggregator<Aggregations, Histogram, Histogram.Bucket> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchEntitiesDescriptionAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.EntitiesDescriptionAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchEntitiesDescriptionAggregator
extends EntitiesDescriptionAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchEntitiesOwnerAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.EntitiesOwnerAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchEntitiesOwnerAggregator
extends EntitiesOwnerAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

8
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchMostActiveUsersAggregator.java
View File

@ -2,10 +2,10 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.MostActiveUsersAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Max;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Max;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchMostActiveUsersAggregator
extends MostActiveUsersAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum, Max> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchMostViewedEntitiesAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.MostViewedEntitiesAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchMostViewedEntitiesAggregator
extends MostViewedEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchPageViewsByEntitiesAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.PageViewsByEntitiesAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchPageViewsByEntitiesAggregator
extends PageViewsByEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchServicesDescriptionAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.ServicesDescriptionAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchServicesDescriptionAggregator
extends ServicesDescriptionAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchServicesOwnerAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.ServicesOwnerAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchServicesOwnerAggregator
extends ServicesOwnerAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchTotalEntitiesAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.TotalEntitiesAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchTotalEntitiesAggregator
extends TotalEntitiesAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchTotalEntitiesByTierAggregator.java
View File

@ -2,9 +2,9 @@ package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import java.util.List;
import org.openmetadata.service.dataInsight.TotalEntitiesByTierAggregator;
import org.opensearch.search.aggregations.Aggregations;
import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.opensearch.search.aggregations.metrics.Sum;
import os.org.opensearch.search.aggregations.Aggregations;
import os.org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
import os.org.opensearch.search.aggregations.metrics.Sum;
public class OpenSearchTotalEntitiesByTierAggregator
extends TotalEntitiesByTierAggregator<Aggregations, MultiBucketsAggregation.Bucket, MultiBucketsAggregation, Sum> {

6
openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/dataInsightAggregator/OpenSearchUnusedAssetsAggregator.java
View File

@ -1,9 +1,9 @@
package org.openmetadata.service.search.opensearch.dataInsightAggregator;
import org.apache.lucene.search.TotalHits;
import org.openmetadata.service.dataInsight.UnusedAssetsAggregator;
import org.opensearch.search.SearchHit;
import org.opensearch.search.SearchHits;
import os.org.apache.lucene.search.TotalHits;
import os.org.opensearch.search.SearchHit;
import os.org.opensearch.search.SearchHits;
public class OpenSearchUnusedAssetsAggregator extends UnusedAssetsAggregator<SearchHits, SearchHit, TotalHits> {
public OpenSearchUnusedAssetsAggregator(SearchHits hits) {

8
openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java
View File

@ -18,8 +18,8 @@ import org.openmetadata.schema.system.StepStats;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.opensearch.action.bulk.BulkItemResponse;
import org.opensearch.action.bulk.BulkResponse;
import os.org.opensearch.action.bulk.BulkItemResponse;
import os.org.opensearch.action.bulk.BulkResponse;
public class ReindexingUtil {
public static final String ENTITY_TYPE_KEY = "entityType";
@ -57,9 +57,9 @@ public class ReindexingUtil {
return success;
}
public static int getSuccessFromBulkResponseEs(org.elasticsearch.action.bulk.BulkResponse response) {
public static int getSuccessFromBulkResponseEs(es.org.elasticsearch.action.bulk.BulkResponse response) {
int success = 0;
for (org.elasticsearch.action.bulk.BulkItemResponse bulkItemResponse : response) {
for (es.org.elasticsearch.action.bulk.BulkItemResponse bulkItemResponse : response) {
if (!bulkItemResponse.isFailed()) {
success++;
}

30
openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/SearchIndexWorkflow.java
View File

@ -20,6 +20,9 @@ import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getT
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.isDataInsightIndex;
import es.org.elasticsearch.action.bulk.BulkItemResponse;
import es.org.elasticsearch.action.bulk.BulkRequest;
import es.org.elasticsearch.action.bulk.BulkResponse;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -28,9 +31,6 @@ import lombok.Getter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.analytics.ReportData;
@ -151,11 +151,11 @@ public class SearchIndexWorkflow implements Runnable {
if (!resultList.getData().isEmpty()) {
if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
// process data to build Reindex Request
org.opensearch.action.bulk.BulkRequest requests =
(org.opensearch.action.bulk.BulkRequest) entityProcessor.process(resultList, contextData);
os.org.opensearch.action.bulk.BulkRequest requests =
(os.org.opensearch.action.bulk.BulkRequest) entityProcessor.process(resultList, contextData);
// process data to build Reindex Request
org.opensearch.action.bulk.BulkResponse response =
(org.opensearch.action.bulk.BulkResponse) searchIndexSink.write(requests, contextData);
os.org.opensearch.action.bulk.BulkResponse response =
(os.org.opensearch.action.bulk.BulkResponse) searchIndexSink.write(requests, contextData);
// update Status
handleErrorsOs(resultList, paginatedEntitiesSource.getLastFailedCursor(), response, currentTime);
// Update stats
@ -222,11 +222,11 @@ public class SearchIndexWorkflow implements Runnable {
if (!resultList.getData().isEmpty()) {
if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
// process data to build Reindex Request
org.opensearch.action.bulk.BulkRequest requests =
(org.opensearch.action.bulk.BulkRequest) dataInsightProcessor.process(resultList, contextData);
os.org.opensearch.action.bulk.BulkRequest requests =
(os.org.opensearch.action.bulk.BulkRequest) dataInsightProcessor.process(resultList, contextData);
// process data to build Reindex Request
org.opensearch.action.bulk.BulkResponse response =
(org.opensearch.action.bulk.BulkResponse) searchIndexSink.write(requests, contextData);
os.org.opensearch.action.bulk.BulkResponse response =
(os.org.opensearch.action.bulk.BulkResponse) searchIndexSink.write(requests, contextData);
handleErrorsOs(resultList, "", response, currentTime);
// Update stats
success = searchRepository.getSearchClient().getSuccessFromBulkResponse(response);
@ -338,7 +338,7 @@ public class SearchIndexWorkflow implements Runnable {
}
private void handleErrorsOs(
ResultList<?> data, String lastCursor, org.opensearch.action.bulk.BulkResponse response, long time) {
ResultList<?> data, String lastCursor, os.org.opensearch.action.bulk.BulkResponse response, long time) {
handleSourceError(data, lastCursor, time);
handleOsSinkErrors(response, time);
}
@ -388,11 +388,11 @@ public class SearchIndexWorkflow implements Runnable {
}
@SneakyThrows
private void handleOsSinkErrors(org.opensearch.action.bulk.BulkResponse response, long time) {
private void handleOsSinkErrors(os.org.opensearch.action.bulk.BulkResponse response, long time) {
List<FailureDetails> details = new ArrayList<>();
for (org.opensearch.action.bulk.BulkItemResponse bulkItemResponse : response) {
for (os.org.opensearch.action.bulk.BulkItemResponse bulkItemResponse : response) {
if (bulkItemResponse.isFailed()) {
org.opensearch.action.bulk.BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
os.org.opensearch.action.bulk.BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
FailureDetails esFailure =
new FailureDetails()
.withContext(

10
openmetadata-service/src/main/resources/elasticsearch/indexMapping.json
View File

@ -131,31 +131,31 @@
"alias": "team",
"parentAliases": []
},
"EntityReportData": {
"entityReportData": {
"indexName": "entity_report_data_index",
"indexMappingFile": "/elasticsearch/entity_report_data_index.json",
"alias": "entityReportData",
"parentAliases": []
},
"WebAnalyticEntityViewReportData": {
"webAnalyticEntityViewReportData": {
"indexName": "web_analytic_entity_view_report_data_index",
"indexMappingFile": "/elasticsearch/web_analytic_entity_view_report_data_index.json",
"alias": "webAnalyticEntityViewReportData",
"parentAliases": []
},
"WebAnalyticUserActivityReportData": {
"webAnalyticUserActivityReportData": {
"indexName": "web_analytic_user_activity_report_data_index",
"indexMappingFile": "/elasticsearch/web_analytic_user_activity_report_data_index.json",
"alias": "webAnalyticUserActivityReportData",
"parentAliases": []
},
"RawCostAnalysisReportData": {
"rawCostAnalysisReportData": {
"indexName": "raw_cost_analysis_report_data_index",
"indexMappingFile": "/elasticsearch/raw_cost_analysis_report_data_index.json",
"alias": "RawCostAnalysisReportData",
"parentAliases": []
},
"AggregatedCostAnalysisReportData": {
"aggregatedCostAnalysisReportData": {
"indexName": "aggregated_cost_analysis_report_data_index",
"indexMappingFile": "/elasticsearch/aggregated_cost_analysis_report_data_index.json",
"alias": "AggregatedCostAnalysisReportData",

2
openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java
View File

@ -16,6 +16,7 @@ package org.openmetadata.service;
import static java.lang.String.format;
import static org.openmetadata.service.util.TablesInitializer.validateAndRunSystemDataMigrations;
import es.org.elasticsearch.client.RestClient;
import io.dropwizard.jersey.jackson.JacksonFeature;
import io.dropwizard.testing.ConfigOverride;
import io.dropwizard.testing.ResourceHelpers;
@ -27,7 +28,6 @@ import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.WebTarget;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.flywaydb.core.Flyway;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.ClientProperties;

32
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -52,6 +52,22 @@ import static org.openmetadata.service.util.TestUtils.UpdateType.NO_CHANGE;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import es.org.elasticsearch.action.search.SearchResponse;
import es.org.elasticsearch.client.Request;
import es.org.elasticsearch.client.Response;
import es.org.elasticsearch.client.RestClient;
import es.org.elasticsearch.search.SearchHit;
import es.org.elasticsearch.search.aggregations.Aggregation;
import es.org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import es.org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import es.org.elasticsearch.search.aggregations.metrics.ParsedTopHits;
import es.org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
import es.org.elasticsearch.xcontent.ContextParser;
import es.org.elasticsearch.xcontent.DeprecationHandler;
import es.org.elasticsearch.xcontent.NamedXContentRegistry;
import es.org.elasticsearch.xcontent.ParseField;
import es.org.elasticsearch.xcontent.XContentParser;
import es.org.elasticsearch.xcontent.json.JsonXContent;
import java.io.IOException;
import java.net.URISyntaxException;
import java.time.Duration;
@ -80,22 +96,6 @@ import org.apache.commons.text.RandomStringGenerator.Builder;
import org.apache.http.client.HttpResponseException;
import org.apache.http.util.EntityUtils;
import org.awaitility.Awaitility;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.metrics.ParsedTopHits;
import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
import org.elasticsearch.xcontent.ContextParser;
import org.elasticsearch.xcontent.DeprecationHandler;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.ParseField;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.AfterAll;

6
openmetadata-service/src/test/java/org/openmetadata/service/resources/analytics/ReportDataResourceTest.java
View File

@ -13,6 +13,9 @@ import static org.openmetadata.service.util.TestUtils.TEST_USER_NAME;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import es.org.elasticsearch.client.Request;
import es.org.elasticsearch.client.Response;
import es.org.elasticsearch.client.RestClient;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
@ -22,9 +25,6 @@ import java.util.Map;
import java.util.UUID;
import javax.ws.rs.client.WebTarget;
import org.apache.http.client.HttpResponseException;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;

29
openmetadata-shaded-deps/elasticsearch-dep/.gitignore vendored Normal file
View File

@ -0,0 +1,29 @@
### IntelliJ IDEA ###
out/
!**/src/main/**/out/
!**/src/test/**/out/
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store

111
openmetadata-shaded-deps/elasticsearch-dep/pom.xml Normal file
View File

@ -0,0 +1,111 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>openmetadata-shaded-deps</artifactId>
<groupId>org.open-metadata</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elasticsearch-deps</artifactId>
<name>Shaded Deps for OpenMetadata Server</name>
<description>Shaded version of dependencies used only for internal OpenMetadata code.</description>
<dependencies>
<!-- All dependencies should be marked as optional to work around
https://issues.apache.org/jira/browse/MSHADE-206 which makes it so
the dependency reduced pom is not reflected in a multi-module build
correctly. -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
<configuration>
<keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<minimizeJar>false</minimizeJar>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<relocations combine.children="append">
<relocation>
<pattern>org.elasticsearch</pattern>
<shadedPattern>es.org.elasticsearch</shadedPattern>
</relocation>
<relocation>
<pattern>org.elasticsearch.client</pattern>
<shadedPattern>es.org.elasticsearch.client</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>es.org.apache.lucene</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.sf</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.dsa</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/*.rsa</exclude>
<exclude>META-INF/*.EC</exclude>
<exclude>META-INF/*.ec</exclude>
<exclude>META-INF/MSFTSIG.SF</exclude>
<exclude>META-INF/MSFTSIG.RSA</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
</excludes>
</filter>
</filters>
</configuration>
</plugin>
<plugin>
<!--
Purpose of build-helper-maven-plugin is to ensure that IntelliJ sees shaded classes.
-->
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.4.0</version>
<executions>
<execution>
<id>workaround-makeItVisibleOnIntellij</id>
<phase>none</phase>
<goals>
<goal>attach-artifact</goal>
</goals>
<configuration>
<artifacts>
<artifact>
<file>${project.build.directory}/${project.build.finalName}.jar</file>
<type>jar</type>
<classifier>optional</classifier>
</artifact>
</artifacts>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

29
openmetadata-shaded-deps/opensearch-dep/.gitignore vendored Normal file
View File

@ -0,0 +1,29 @@
### IntelliJ IDEA ###
out/
!**/src/main/**/out/
!**/src/test/**/out/
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store

111
openmetadata-shaded-deps/opensearch-dep/pom.xml Normal file
View File

@ -0,0 +1,111 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>openmetadata-shaded-deps</artifactId>
<groupId>org.open-metadata</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>opensearch-deps</artifactId>
<name>OpenSearch Shaded Deps for OpenMetadata Server</name>
<description>Shaded version of OpenSearch dependencies used only for internal OpenMetadata code.</description>
<dependencies>
<!-- All dependencies should be marked as optional to work around
https://issues.apache.org/jira/browse/MSHADE-206 which makes it so
the dependency reduced pom is not reflected in a multi-module build
correctly. -->
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-high-level-client</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
<configuration>
<keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<minimizeJar>false</minimizeJar>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<relocations combine.children="append">
<relocation>
<pattern>org.opensearch</pattern>
<shadedPattern>os.org.opensearch</shadedPattern>
</relocation>
<relocation>
<pattern>org.opensearch.client</pattern>
<shadedPattern>os.org.opensearch.client</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.lucene</pattern>
<shadedPattern>os.org.apache.lucene</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.sf</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.dsa</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/*.rsa</exclude>
<exclude>META-INF/*.EC</exclude>
<exclude>META-INF/*.ec</exclude>
<exclude>META-INF/MSFTSIG.SF</exclude>
<exclude>META-INF/MSFTSIG.RSA</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
</excludes>
</filter>
</filters>
</configuration>
</plugin>
<plugin>
<!--
Purpose of build-helper-maven-plugin is to ensure that IntelliJ sees shaded classes.
-->
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.4.0</version>
<executions>
<execution>
<id>workaround-makeItVisibleOnIntellij</id>
<phase>none</phase>
<goals>
<goal>attach-artifact</goal>
</goals>
<configuration>
<artifacts>
<artifact>
<file>${project.build.directory}/${project.build.finalName}.jar</file>
<type>jar</type>
<classifier>optional</classifier>
</artifact>
</artifacts>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

19
openmetadata-shaded-deps/pom.xml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>platform</artifactId>
<groupId>org.open-metadata</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>openmetadata-shaded-deps</artifactId>
<name>Shaded Deps for OpenMetadata Server</name>
<description>Shaded version of dependencies used only for internal OpenMetadata code.</description>
<modules>
<module>elasticsearch-dep</module>
<module>opensearch-dep</module>
</modules>
<packaging>pom</packaging>
</project>

51
pom.xml
View File

@ -31,6 +31,7 @@
<modules>
<module>openmetadata-spec</module>
<module>common</module>
<module>openmetadata-shaded-deps</module>
<module>openmetadata-service</module>
<module>openmetadata-ui</module>
<module>openmetadata-dist</module>
@ -107,8 +108,8 @@
<!-- We need ElasticSearch client to be compatible with both ElasticSearch and AWS OpenSearch
This compatibility broken in 7.14, so lets keep this version pinned to 7.13.x
-->
<elasticsearch.version>7.16.3</elasticsearch.version>
<opensearch.version>1.3.0</opensearch.version>
<elasticsearch.version>7.17.13</elasticsearch.version>
<opensearch.version>2.5.0</opensearch.version>
<httpasyncclient.version>4.1.5</httpasyncclient.version>
<openapiswagger.version>2.2.15</openapiswagger.version>
<spring-security-kerberos-core.version>1.0.1.RELEASE</spring-security-kerberos-core.version>
@ -286,49 +287,17 @@
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${elasticsearch.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-grouping</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-memory</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial-extras</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial3d</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-suggest</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-high-level-client</artifactId>
<version>${opensearch.version}</version>
<!-- <classifier>shaded</classifier>-->
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${elasticsearch.version}</version>
<!-- <classifier>shaded</classifier>-->
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>