diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index f18452b8245..c66899dec68 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -17,9 +17,11 @@ import static org.openmetadata.service.Entity.ORGANIZATION_NAME; import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; +import com.fasterxml.jackson.core.type.TypeReference; import java.io.IOException; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -86,7 +88,9 @@ import org.openmetadata.schema.settings.SettingsType; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestDefinition; import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.SQLQuery; import org.openmetadata.schema.type.TagCategory; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TaskStatus; @@ -394,6 +398,71 @@ public interface CollectionDAO { @SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension") String getExtension(@Bind("id") String id, @Bind("extension") String extension); + @ConnectionAwareSqlQuery( + value = + "select count(*) from entity_extension d, json_table(d.json, '$[*]' columns (" + + " vote double path '$.vote', " + + " query varchar(200) path '$.query'," + + " users json path '$.users'," + + " checksum varchar(200) path '$.checksum'," + + " duration double path '$.duration'," + + " queryDate varchar(200) path '$.queryDate'" + + " )" + + "" + + ") as j where d.id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select count(*) from entity_extension as ee , jsonb_to_recordset(ee.json) as x (vote decimal,query varchar,users json,checksum varchar,duration decimal,queryDate varchar) where ee.id = :id ", + connectionType = POSTGRES) + int getTotalQueriesCount(@Bind("id") String id); + + @RegisterRowMapper(SqlQueryMapper.class) + @ConnectionAwareSqlQuery( + value = + "select j.* from entity_extension d, json_table(d.json, '$[*]' columns (" + + " vote Double path '$.vote', " + + " query varchar(200) path '$.query'," + + " users json path '$.users'," + + " checksum varchar(200) path '$.checksum'," + + " duration Double path '$.duration'," + + " queryDate varchar(200) path '$.queryDate'" + + " )" + + ") as j where d.id = :id AND d.extension = :extension And query > :after order by query LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select x.* from entity_extension as ee , jsonb_to_recordset(ee.json) as x (vote decimal,query varchar,users json,checksum varchar,duration decimal,queryDate varchar) where ee.id = :id and ee.extension = :extension and query > :after order by query LIMIT :limit ", + connectionType = POSTGRES) + List getExtensionPagination( + @Bind("id") String id, + @Bind("extension") String extension, + @Bind("limit") int limit, + @Bind("after") String after); + + @RegisterRowMapper(SqlQueryMapper.class) + @ConnectionAwareSqlQuery( + value = + "select j.* from entity_extension d, json_table(d.json, '$[*]' columns (" + + " vote Double path '$.vote'," + + " query varchar(200) path '$.query'," + + " users json path '$.users'," + + " checksum varchar(200) path '$.checksum'," + + " duration Double path '$.duration'," + + " queryDate varchar(200) path '$.queryDate'" + + " )" + + ") as j where d.id = :id AND d.extension = :extension And query < :before order by query LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select x.* from entity_extension as ee , jsonb_to_recordset(ee.json) as x (vote decimal,query varchar,users json,checksum varchar,duration decimal,queryDate varchar) where ee.id = :id and ee.extension = :extension and query < : before order by query LIMIT :limit ", + connectionType = POSTGRES) + List listBeforeTableQueries( + @Bind("id") String id, + @Bind("extension") String extension, + @Bind("limit") int limit, + @Bind("before") String before); + @RegisterRowMapper(ExtensionMapper.class) @SqlQuery( "SELECT extension, json FROM entity_extension WHERE id = :id AND extension " @@ -438,6 +507,26 @@ public interface CollectionDAO { } } + class SqlQueryMapper implements RowMapper { + @Override + public SQLQuery map(ResultSet rs, StatementContext ctx) throws SQLException { + List users = new ArrayList<>(); + String json = rs.getString("users"); + try { + users = JsonUtils.readValue(json, new TypeReference>() {}); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new SQLQuery() + .withVote(rs.getDouble("vote")) + .withQuery(rs.getString("query")) + .withUsers(users) + .withChecksum(rs.getString("checksum")) + .withDuration(rs.getDouble("duration")) + .withQueryDate(rs.getString("queryDate")); + } + } + @Getter @Builder class EntityRelationshipRecord { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index e8ba891eeac..154878c66f1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -519,6 +519,58 @@ public class TableRepository extends EntityRepository { return table.withTableQueries(getQueries(table)); } + public ResultList getQueriesForPagination(UUID id, int limit, String before, String after) { + RestUtil.validateCursors(before, after); + int total = daoCollection.entityExtensionDAO().getTotalQueriesCount(id.toString()); + List tableQueries = null; + if (before != null) { + tableQueries = + daoCollection + .entityExtensionDAO() + .listBeforeTableQueries(id.toString(), "table.tableQueries", limit + 1, RestUtil.decodeCursor(before)); + } else { + tableQueries = + daoCollection + .entityExtensionDAO() + .getExtensionPagination( + id.toString(), "table.tableQueries", limit + 1, after == null ? "" : RestUtil.decodeCursor(after)); + } + ResultList tablQueriesList; + if (before != null) { + tablQueriesList = getBeforeListForTableQueries(tableQueries, limit, total); + } else { + tablQueriesList = getAfterListForTableQueries(after, tableQueries, limit, total); + } + return tablQueriesList; + } + + private ResultList getBeforeListForTableQueries(List tableQueries, int limit, int total) { + String beforeCursor = null; + String afterCursor; + if (tableQueries.size() > limit) { // If extra result exists, then previous page exists - return before cursor + tableQueries.remove(0); + beforeCursor = tableQueries.get(0).getQuery(); + } + afterCursor = tableQueries.get(tableQueries.size() - 1).getQuery(); + return getQueryResultList(tableQueries, beforeCursor, afterCursor, total); + } + + private ResultList getAfterListForTableQueries( + String after, List tableQueries, int limit, int total) { + String beforeCursor; + String afterCursor = null; + beforeCursor = after == null ? null : tableQueries.get(0).getQuery(); + if (tableQueries.size() > limit) { // If extra result exists, then next page exists - return after cursor + tableQueries.remove(limit); + afterCursor = tableQueries.get(limit - 1).getQuery(); + } + return getQueryResultList(tableQueries, beforeCursor, afterCursor, total); + } + + private ResultList getQueryResultList(List queries, String before, String after, int total) { + return new ResultList<>(queries, before, after, total); + } + private List getQueries(Table table) throws IOException { List tableQueries = JsonUtils.readObjects( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index d9f0745f0c7..da474aded7e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -895,6 +895,38 @@ public class TableResource extends EntityResource { return addHref(uriInfo, table); } + @GET + @Path("/{id}/getTableQueries") + @Operation( + operationId = "getTableQueryList", + summary = "get table query data", + tags = "tables", + description = "get table query data from the table.", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = Table.class))) + }) + public ResultList getTableQueryList( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the table", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Valid SQLQuery sqlQuery, + @DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, + @Parameter(description = "Returns list of users before this cursor", schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter(description = "Returns list of users after this cursor", schema = @Schema(type = "string")) + @QueryParam("after") + String after) + throws IOException { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.VIEW_QUERIES); + authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); + ResultList getTableQueryList = dao.getQueriesForPagination(id, limitParam, before, after); + return getTableQueryList; + } + @PUT @Path("/{id}/dataModel") @Operation(