Fix #16788: Optimize feed query performance issues introduced in 1.4.2 (#16862)

bootstrap/sql/migrations/native/1.4.4/mysql/schemaChanges.sql

@@ -1 +1,13 @@
 ALTER TABLE user_entity ADD UNIQUE (name);
+
+-- Adding a new column 'taskAssigneesIds' to improve query performance by avoiding the use of JSON_EXTRACT in queries
+-- This column is generated to store the 'id' values extracted from the 'taskAssignees' JSON array, which helps in optimizing search and filtering operations.
+ALTER TABLE thread_entity
+ADD COLUMN taskAssigneesIds TEXT GENERATED ALWAYS AS (JSON_EXTRACT(taskAssignees, '$[*].id'));
+ALTER TABLE thread_entity ADD INDEX taskAssigneesIds_index (taskAssigneesIds(700));
+
+-- Creating an index on (type, resolved, updatedAt) to optimize query performance for filtering by these columns for activity feed.-FeedDao.List
+CREATE INDEX thread_type_resolved_updatedAt_index ON thread_entity (type, resolved, updatedAt);
+
+-- Creating an index on (createdAt) to improve performance for queries that sort or filter by creation date in activity feed.-FeedDao.List
+CREATE INDEX created_at_index ON thread_entity (createdAt);

bootstrap/sql/migrations/native/1.4.4/postgres/schemaChanges.sql

@@ -1 +1,18 @@
 ALTER TABLE user_entity ADD UNIQUE (name);
+
+-- Adding a new column 'taskAssigneesIds' to improve query performance by avoiding the use of JSON_EXTRACT in queries
+-- This column is generated to store the 'id' values extracted from the 'taskAssignees' JSON array, which helps in optimizing search and filtering operations.
+ALTER TABLE thread_entity
+ADD COLUMN taskAssigneesIds TEXT;
+UPDATE thread_entity
+SET taskAssigneesIds = (
+    SELECT jsonb_agg(elem->>'id')
+    FROM jsonb_array_elements(taskAssignees) AS elem
+);
+CREATE INDEX taskAssigneesIds_index ON thread_entity(taskAssigneesIds);
+
+-- Creating an index on (type, resolved, updatedAt) to optimize query performance for filtering by these columns for activity feed.-FeedDao.List
+CREATE INDEX thread_type_resolved_updatedAt_index ON thread_entity (type, resolved, updatedAt);
+
+-- Creating an index on (createdAt) to improve performance for queries that sort or filter by creation date in activity feed.-FeedDao.List
+CREATE INDEX created_at_index ON thread_entity (createdAt);

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java

@@ -1012,7 +1012,7 @@ public interface CollectionDAO {
     @ConnectionAwareSqlQuery(
         value =
             "SELECT json FROM thread_entity <condition> AND "
-                + "JSON_OVERLAPS(json_extract(taskAssignees, '$[*].id'), :userTeamJsonMysql) "
+                + "JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) "
                 + "ORDER BY createdAt DESC "
                 + "LIMIT :limit",
         connectionType = MYSQL)
@@ -1030,7 +1030,7 @@ public interface CollectionDAO {
     @ConnectionAwareSqlQuery(
         value =
             "SELECT count(id) FROM thread_entity <condition> AND "
-                + "JSON_OVERLAPS(json_extract(taskAssignees, '$[*].id'), :userTeamJsonMysql) ",
+                + "JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) ",
         connectionType = MYSQL)
     int listCountTasksAssignedTo(
         @BindList("userTeamJsonPostgres") List<String> userTeamJsonPostgres,
@@ -1047,7 +1047,7 @@ public interface CollectionDAO {
     @ConnectionAwareSqlQuery(
         value =
             "SELECT json FROM thread_entity <condition> "
-                + "AND (JSON_OVERLAPS(JSON_EXTRACT(taskAssignees, '$[*].id'), :userTeamJsonMysql) OR createdBy = :username) "
+                + "AND (JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) OR createdBy = :username) "
                 + "ORDER BY createdAt DESC "
                 + "LIMIT :limit",
         connectionType = MYSQL)
@@ -1066,7 +1066,7 @@ public interface CollectionDAO {
     @ConnectionAwareSqlQuery(
         value =
             "SELECT count(id) FROM thread_entity <condition> "
-                + "AND (JSON_OVERLAPS(JSON_EXTRACT(taskAssignees, '$[*].id'), :userTeamJsonMysql) OR createdBy = :username) ",
+                + "AND (JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) OR createdBy = :username) ",
         connectionType = MYSQL)
     int listCountTasksOfUser(
         @BindList("userTeamJsonPostgres") List<String> userTeamJsonPostgres,
@@ -1245,7 +1245,7 @@ public interface CollectionDAO {
                 + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2))) "
                 + " OR id in (SELECT toId FROM entity_relationship WHERE ((fromEntity='user' AND fromId= :userId) OR "
                 + "(fromEntity='team' AND fromId IN (<teamIds>))) AND relation=11)) OR "
-                + "(JSON_OVERLAPS(JSON_EXTRACT(taskAssignees, '$[*].id'), :userTeamJsonMysql) OR createdBy = :username)"
+                + "(JSON_OVERLAPS(taskAssigneesIds, :userTeamJsonMysql) OR createdBy = :username)"
                 + " GROUP BY te.type, te.taskStatus",
         connectionType = MYSQL)
     @RegisterRowMapper(OwnerCountFieldMapper.class)