yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-02 02:26:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #16788: Optimize feed query performance issues introduced in 1.4.2 (#16862)

Browse Source
This commit is contained in:
sonika-shah 2024-07-02 08:28:47 +05:30 committed by GitHub
parent eac7021af0
commit be243c5cea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 36 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bootstrap/sql/migrations/native/1.4.4/mysql/schemaChanges.sql
View File

@ -1 +1,13 @@
ALTER TABLE user_entity ADD UNIQUE (name);
ALTER TABLE user_entity ADD UNIQUE (name);
-- Adding a new column 'taskAssigneesIds' to improve query performance by avoiding the use of JSON_EXTRACT in queries
-- This column is generated to store the 'id' values extracted from the 'taskAssignees' JSON array, which helps in optimizing search and filtering operations.
ALTER TABLE thread_entity
ADD COLUMN taskAssigneesIds TEXT GENERATED ALWAYS AS (JSON_EXTRACT(taskAssignees, '$[*].id'));
ALTER TABLE thread_entity ADD INDEX taskAssigneesIds_index (taskAssigneesIds(700));
-- Creating an index on (type, resolved, updatedAt) to optimize query performance for filtering by these columns for activity feed.-FeedDao.List
CREATE INDEX thread_type_resolved_updatedAt_index ON thread_entity (type, resolved, updatedAt);
-- Creating an index on (createdAt) to improve performance for queries that sort or filter by creation date in activity feed.-FeedDao.List
CREATE INDEX created_at_index ON thread_entity (createdAt);

19
bootstrap/sql/migrations/native/1.4.4/postgres/schemaChanges.sql
View File

@ -1 +1,18 @@
ALTER TABLE user_entity ADD UNIQUE (name);
ALTER TABLE user_entity ADD UNIQUE (name);
-- Adding a new column 'taskAssigneesIds' to improve query performance by avoiding the use of JSON_EXTRACT in queries
-- This column is generated to store the 'id' values extracted from the 'taskAssignees' JSON array, which helps in optimizing search and filtering operations.
ALTER TABLE thread_entity
ADD COLUMN taskAssigneesIds TEXT;
UPDATE thread_entity
SET taskAssigneesIds = (
SELECT jsonb_agg(elem->>'id')
FROM jsonb_array_elements(taskAssignees) AS elem
);
CREATE INDEX taskAssigneesIds_index ON thread_entity(taskAssigneesIds);
-- Creating an index on (type, resolved, updatedAt) to optimize query performance for filtering by these columns for activity feed.-FeedDao.List
CREATE INDEX thread_type_resolved_updatedAt_index ON thread_entity (type, resolved, updatedAt);
-- Creating an index on (createdAt) to improve performance for queries that sort or filter by creation date in activity feed.-FeedDao.List
CREATE INDEX created_at_index ON thread_entity (createdAt);

10
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -1012,7 +1012,7 @@ public interface CollectionDAO {
@ConnectionAwareSqlQuery(
value =
"SELECT json FROM thread_entity <condition> AND "
+ "JSON_OVERLAPS(json_extract(taskAssignees, '$[*].id'), :userTeamJsonMysql) "
+ "JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) "
+ "ORDER BY createdAt DESC "
+ "LIMIT :limit",
connectionType = MYSQL)
@ -1030,7 +1030,7 @@ public interface CollectionDAO {
@ConnectionAwareSqlQuery(
value =
"SELECT count(id) FROM thread_entity <condition> AND "
+ "JSON_OVERLAPS(json_extract(taskAssignees, '$[*].id'), :userTeamJsonMysql) ",
+ "JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) ",
connectionType = MYSQL)
int listCountTasksAssignedTo(
@BindList("userTeamJsonPostgres") List<String> userTeamJsonPostgres,
@ -1047,7 +1047,7 @@ public interface CollectionDAO {
@ConnectionAwareSqlQuery(
value =
"SELECT json FROM thread_entity <condition> "
+ "AND (JSON_OVERLAPS(JSON_EXTRACT(taskAssignees, '$[*].id'), :userTeamJsonMysql) OR createdBy = :username) "
+ "AND (JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) OR createdBy = :username) "
+ "ORDER BY createdAt DESC "
+ "LIMIT :limit",
connectionType = MYSQL)
@ -1066,7 +1066,7 @@ public interface CollectionDAO {
@ConnectionAwareSqlQuery(
value =
"SELECT count(id) FROM thread_entity <condition> "
+ "AND (JSON_OVERLAPS(JSON_EXTRACT(taskAssignees, '$[*].id'), :userTeamJsonMysql) OR createdBy = :username) ",
+ "AND (JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) OR createdBy = :username) ",
connectionType = MYSQL)
int listCountTasksOfUser(
@BindList("userTeamJsonPostgres") List<String> userTeamJsonPostgres,
@ -1245,7 +1245,7 @@ public interface CollectionDAO {
+ "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2))) "
+ " OR id in (SELECT toId FROM entity_relationship WHERE ((fromEntity='user' AND fromId= :userId) OR "
+ "(fromEntity='team' AND fromId IN (<teamIds>))) AND relation=11)) OR "
+ "(JSON_OVERLAPS(JSON_EXTRACT(taskAssignees, '$[*].id'), :userTeamJsonMysql) OR createdBy = :username)"
+ "(JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) OR createdBy = :username)"
+ " GROUP BY te.type, te.taskStatus",
connectionType = MYSQL)
@RegisterRowMapper(OwnerCountFieldMapper.class)