From fa68a1f18f0275d1b3eecfb14eb85d5f5123f269 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Wed, 2 Nov 2022 14:07:41 +0530 Subject: [PATCH] Fix #6754: Added User Details for Usage (#8449) --- .../ingestion/processor/query_parser.py | 1 + .../metadata/ingestion/stage/table_usage.py | 38 ++++++++++++++++++- .../service/jdbi3/TableRepository.java | 9 +++++ .../json/schema/entity/data/table.json | 9 +++-- .../json/schema/type/queryParserData.json | 4 ++ .../TableQueries/QueryCard.test.tsx | 24 ++++++------ .../src/components/TableQueries/QueryCard.tsx | 10 ++--- 7 files changed, 74 insertions(+), 21 deletions(-) diff --git a/ingestion/src/metadata/ingestion/processor/query_parser.py b/ingestion/src/metadata/ingestion/processor/query_parser.py index aaffee72bb5..ecbba14008b 100644 --- a/ingestion/src/metadata/ingestion/processor/query_parser.py +++ b/ingestion/src/metadata/ingestion/processor/query_parser.py @@ -74,6 +74,7 @@ def parse_sql_statement(record: TableQuery) -> Optional[ParsedData]: databaseName=record.databaseName, databaseSchema=record.databaseSchema, sql=record.query, + userName=record.userName, date=start_date.__root__.strftime("%Y-%m-%d"), serviceName=record.serviceName, ) diff --git a/ingestion/src/metadata/ingestion/stage/table_usage.py b/ingestion/src/metadata/ingestion/stage/table_usage.py index 48d1b3f2a2d..ad8a0b3984c 100644 --- a/ingestion/src/metadata/ingestion/stage/table_usage.py +++ b/ingestion/src/metadata/ingestion/stage/table_usage.py @@ -23,9 +23,12 @@ from metadata.generated.schema.entity.data.table import SqlQuery from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) +from metadata.generated.schema.entity.teams.user import User +from metadata.generated.schema.type.entityReference import EntityReference from metadata.generated.schema.type.queryParserData import QueryParserData from metadata.generated.schema.type.tableUsageCount import TableUsageCount from metadata.ingestion.api.stage import Stage, StageStatus +from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils.constants import UTF_8 from metadata.utils.logger import ingestion_logger @@ -54,6 +57,7 @@ class TableUsageStage(Stage[QueryParserData]): ): self.config = config self.metadata_config = metadata_config + self.metadata = OpenMetadata(self.metadata_config) self.status = StageStatus() self.table_usage = {} self.table_queries = {} @@ -70,11 +74,41 @@ class TableUsageStage(Stage[QueryParserData]): config = TableStageConfig.parse_obj(config_dict) return cls(config, metadata_config) + def _get_user_entity(self, username: str): + if username: + user = self.metadata.get_by_name(entity=User, fqn=username) + if user: + return [ + EntityReference( + id=user.id, + type="user", + name=user.name.__root__, + fullyQualifiedName=user.fullyQualifiedName.__root__, + description=user.description, + displayName=user.displayName, + deleted=user.deleted, + href=user.href, + ) + ] + return [] + def _add_sql_query(self, record, table): if self.table_queries.get((table, record.date)): - self.table_queries[(table, record.date)].append(SqlQuery(query=record.sql)) + self.table_queries[(table, record.date)].append( + SqlQuery( + query=record.sql, + users=self._get_user_entity(record.userName), + queryDate=record.date, + ) + ) else: - self.table_queries[(table, record.date)] = [SqlQuery(query=record.sql)] + self.table_queries[(table, record.date)] = [ + SqlQuery( + query=record.sql, + users=self._get_user_entity(record.userName), + queryDate=record.date, + ) + ] def stage_record(self, record: QueryParserData) -> None: if not record or not record.parsedData: diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 0ff8ae820ef..d19cfad08d3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -380,6 +381,14 @@ public class TableRepository extends EntityRepository { storedMapQueries.put(q.getChecksum(), q); } } + SQLQuery oldQuery = storedMapQueries.get(query.getChecksum()); + if (oldQuery != null && query.getUsers() != null) { + // Merge old and new users + List userList = query.getUsers(); + userList.addAll(oldQuery.getUsers()); + HashSet userSet = new HashSet<>(userList); + query.setUsers(new ArrayList<>(userSet)); + } storedMapQueries.put(query.getChecksum(), query); List updatedQueries = new ArrayList<>(storedMapQueries.values()); daoCollection diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index 8bbe6203b0f..687d422dffd 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -578,9 +578,12 @@ "description": "How long did the query took to run in seconds.", "type": "number" }, - "user": { - "description": "User who ran this query.", - "$ref": "../../type/entityReference.json", + "users": { + "description": "List of users who ran this query.", + "type":"array", + "items": { + "$ref": "../../type/entityReference.json" + }, "default": null }, "vote": { diff --git a/openmetadata-spec/src/main/resources/json/schema/type/queryParserData.json b/openmetadata-spec/src/main/resources/json/schema/type/queryParserData.json index ea311a81c23..f99a07af78f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/queryParserData.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/queryParserData.json @@ -38,6 +38,10 @@ "description": "Name that identifies this database service.", "type": "string" }, + "userName": { + "description": "Name of the user that executed the SQL query", + "type": "string" + }, "date": { "description": "Date of execution of SQL query", "type": "string" diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryCard.test.tsx index c8eb3a6982d..24c228bc6fe 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryCard.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryCard.test.tsx @@ -14,7 +14,7 @@ import { findByTestId, findByText, - getByTestId, + // getByTestId, queryByTestId, render, } from '@testing-library/react'; @@ -25,13 +25,15 @@ import QueryCard from './QueryCard'; const mockQueryData = { query: 'select products from raw_product_catalog', duration: 0.309, - user: { - id: 'd4785e53-bbdb-4dbd-b368-009fdb50c2c6', - type: 'user', - name: 'aaron_johnson0', - displayName: 'Aaron Johnson', - href: 'http://localhost:8585/api/v1/users/d4785e53-bbdb-4dbd-b368-009fdb50c2c6', - }, + users: [ + { + id: 'd4785e53-bbdb-4dbd-b368-009fdb50c2c6', + type: 'user', + name: 'aaron_johnson0', + displayName: 'Aaron Johnson', + href: 'http://localhost:8585/api/v1/users/d4785e53-bbdb-4dbd-b368-009fdb50c2c6', + }, + ], vote: 1, checksum: '0232b0368458aadb29230ccc531462c9', }; @@ -49,7 +51,7 @@ describe('Test QueryCard Component', () => { const { container } = render(, { wrapper: MemoryRouter, }); - const queryHeader = getByTestId(container, 'query-header'); + // const queryHeader = getByTestId(container, 'query-header'); const query = await findByText(container, /SchemaEditor/i); const copyQueryButton = await findByText( container, @@ -61,7 +63,7 @@ describe('Test QueryCard Component', () => { 'expand-collapse-button' ); - expect(queryHeader).toBeInTheDocument(); + // expect(queryHeader).toBeInTheDocument(); expect(query).toBeInTheDocument(); expect(copyQueryButton).toBeInTheDocument(); expect(expandButton).toBeInTheDocument(); @@ -69,7 +71,7 @@ describe('Test QueryCard Component', () => { it('Should not render header if user is undefined', async () => { const { container } = render( - , + , { wrapper: MemoryRouter, } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryCard.tsx index 04f5fe6b69c..255cca44208 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryCard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableQueries/QueryCard.tsx @@ -12,10 +12,10 @@ */ import classNames from 'classnames'; -import { isUndefined } from 'lodash'; +// import { isUndefined } from 'lodash'; import React, { FC, HTMLAttributes, useState } from 'react'; -import { Link } from 'react-router-dom'; -import { getUserPath } from '../../constants/constants'; +// import { Link } from 'react-router-dom'; +// import { getUserPath } from '../../constants/constants'; import { CSMode } from '../../enums/codemirror.enum'; import { SQLQuery } from '../../generated/entity/data/table'; import SVGIcons, { Icons } from '../../utils/SvgUtils'; @@ -34,7 +34,7 @@ const QueryCard: FC = ({ className, query }) => { className )} onClick={() => setExpanded((pre) => !pre)}> - {!isUndefined(query.user) && !isUndefined(query.duration) ? ( + {/* {!isUndefined(query.user) && !isUndefined(query.duration) ? (

Last run by{' '} @@ -49,7 +49,7 @@ const QueryCard: FC = ({ className, query }) => { {query.duration} seconds

- ) : null} + ) : null} */}