yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-16 12:33:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve count/feed api performance for 1.5 (#17576)

Browse Source
This commit is contained in:
sonika-shah 2024-08-23 23:50:34 +05:30 committed by GitHub
parent d81a57c44e
commit cce22a1b5c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 29 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bootstrap/sql/migrations/native/1.5.0/mysql/schemaChanges.sql
View File

@ -329,3 +329,8 @@ ADD COLUMN taskAssigneesIds TEXT GENERATED ALWAYS AS (
) STORED; ) STORED;
CREATE FULLTEXT INDEX taskAssigneesIds_index ON thread_entity(taskAssigneesIds); CREATE FULLTEXT INDEX taskAssigneesIds_index ON thread_entity(taskAssigneesIds);
-- Add indexes on thread_entity and entity_relationship to improve count/feed api performance
CREATE INDEX idx_thread_entity_entityId_createdAt ON thread_entity (entityId, createdAt);
CREATE INDEX idx_thread_entity_id_type_status ON thread_entity (id, type, taskStatus);
CREATE INDEX idx_er_fromEntity_fromId_toEntity_relation ON entity_relationship (fromEntity, fromId, toEntity, relation);

6
bootstrap/sql/migrations/native/1.5.0/postgres/schemaChanges.sql
View File

@ -299,3 +299,9 @@ ADD COLUMN taskAssigneesIds TEXT GENERATED ALWAYS AS (
CREATE INDEX idx_task_assignees_ids_fulltext CREATE INDEX idx_task_assignees_ids_fulltext
ON thread_entity USING GIN (to_tsvector('simple', taskAssigneesIds)); ON thread_entity USING GIN (to_tsvector('simple', taskAssigneesIds));
-- Add indexes on thread_entity and entity_relationship to improve count/feed api performance
CREATE INDEX idx_thread_entity_entityId_createdAt ON thread_entity (entityId, createdAt);
CREATE INDEX idx_thread_entity_id_type_status ON thread_entity (id, type, taskStatus);
CREATE INDEX idx_er_fromEntity_fromId_toEntity_relation ON entity_relationship (fromEntity, fromId, toEntity, relation);

37
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -1594,36 +1594,35 @@ public interface CollectionDAO {
} }
@SqlQuery( @SqlQuery(
"SELECT entityLink, type, taskStatus, COUNT(id) count " "SELECT entityLink, type, taskStatus, COUNT(id) as count "
+ "FROM ( " + "FROM ( "
+ " SELECT te.entityLink, te.type, te.taskStatus, te.id " + " SELECT te.entityLink, te.type, te.taskStatus, te.id "
+ " FROM thread_entity te " + " FROM thread_entity te "
+ " WHERE te.entityId = :entityId " + " WHERE te.entityId = :entityId "
+ " OR EXISTS ( " + " UNION "
+ " SELECT 1 " + " SELECT te.entityLink, te.type, te.taskStatus, te.id "
+ " FROM field_relationship fr " + " FROM thread_entity te "
+ " WHERE fr.fromFQNHash = MD5(te.id) " + " WHERE te.hash_id IN ( "
+ " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fr.toFQNHash = :fqnPrefixHash) " + " SELECT fr.fromFQNHash "
+ " FROM field_relationship fr "
+ " WHERE (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fr.toFQNHash = :fqnPrefixHash) "
+ " AND fr.fromType = 'THREAD' " + " AND fr.fromType = 'THREAD' "
+ " AND (:toType1 IS NULL OR fr.toType LIKE CONCAT(:toType1, '.%') OR fr.toType = :toType1) " + " AND (:toType1 IS NULL OR fr.toType LIKE CONCAT(:toType1, '.%') OR fr.toType = :toType1) "
+ " AND fr.relation = 3 " + " AND fr.relation = 3 "
+ " ) " + " ) "
+ " UNION ALL " + " UNION "
+ " SELECT te.entityLink, te.type, te.taskStatus, te.id " + " SELECT te.entityLink, te.type, te.taskStatus, te.id "
+ " FROM thread_entity te " + " FROM thread_entity te "
+ " WHERE te.type = 'Task' " + " WHERE te.type = 'Task' "
+ " AND ( " + " AND te.hash_id IN ( "
+ " EXISTS ( " + " SELECT fr.fromFQNHash "
+ " SELECT 1 " + " FROM field_relationship fr "
+ " FROM field_relationship fr " + " JOIN thread_entity te2 ON te2.hash_id = fr.fromFQNHash WHERE fr.fromFQNHash = te.hash_id AND te2.type = 'Task' "
+ " JOIN thread_entity te2 ON MD5(te2.id) = fr.fromFQNHash "
+ " WHERE fr.fromFQNHash = MD5(te.id) "
+ " AND te2.type = 'Task' "
+ " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fr.toFQNHash = :fqnPrefixHash) " + " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fr.toFQNHash = :fqnPrefixHash) "
+ " AND fr.fromType = 'THREAD' " + " AND fr.fromType = 'THREAD' "
+ " AND (:toType2 IS NULL OR fr.toType LIKE CONCAT(:toType2, '.%') OR fr.toType = :toType2) " + " AND (:toType2 IS NULL OR fr.toType LIKE CONCAT(:toType2, '.%') OR fr.toType = :toType2) "
+ " AND fr.relation = 3 " + " AND fr.relation = 3 "
+ " )) " + " ) "
+ ") AS combined_results " + ") AS combined_results "
+ "GROUP BY entityLink, type, taskStatus ") + "GROUP BY entityLink, type, taskStatus ")
@RegisterRowMapper(ThreadCountFieldMapper.class) @RegisterRowMapper(ThreadCountFieldMapper.class)