yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 18:07:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(web) execution request details modal prioritizes stats from ingestion report (#13161)

Browse Source
This commit is contained in:
Jay 2025-04-10 12:34:07 -04:00 committed by GitHub
parent 2504d28255
commit 7365ac6c64
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 255 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
datahub-web-react/src/app/ingest/source/IngestedAssets.tsx
View File

@ -1,6 +1,7 @@
import { Button, Typography } from 'antd';
import React, { useState } from 'react';
import styled from 'styled-components';
import { Maybe, ExecutionRequestResult } from '@src/types.generated';
import { useGetSearchResultsForMultipleQuery } from '../../../graphql/search.generated';
import { EmbeddedListSearchModal } from '../../entity/shared/components/styled/search/EmbeddedListSearchModal';
import { ANTD_GRAY } from '../../entity/shared/constants';
@ -8,7 +9,7 @@ import { UnionType } from '../../search/utils/constants';
import { formatNumber } from '../../shared/formatNumber';
import { Message } from '../../shared/Message';
import { useEntityRegistry } from '../../useEntityRegistry';
import { extractEntityTypeCountsFromFacets } from './utils';
import { extractEntityTypeCountsFromFacets, getEntitiesIngestedByType, getTotalEntitiesIngested } from './utils';
const HeaderContainer = styled.div`
display: flex;
@ -51,19 +52,27 @@ const ViewAllButton = styled(Button)`
type Props = {
id: string;
executionResult?: Maybe<Partial<ExecutionRequestResult>>;
};
const ENTITY_FACET_NAME = 'entity';
const TYPE_NAMES_FACET_NAME = 'typeNames';
export default function IngestedAssets({ id }: Props) {
export default function IngestedAssets({ id, executionResult }: Props) {
const entityRegistry = useEntityRegistry();
// First thing to do is to search for all assets with the id as the run id!
const [showAssetSearch, setShowAssetSearch] = useState(false);
// Try getting the counts via the ingestion report.
const totalEntitiesIngested = executionResult && getTotalEntitiesIngested(executionResult);
const entitiesIngestedByTypeFromReport = executionResult && getEntitiesIngestedByType(executionResult);
// Fallback to the search across entities.
// First thing to do is to search for all assets with the id as the run id!
// Execute search
const { data, loading, error } = useGetSearchResultsForMultipleQuery({
skip: totalEntitiesIngested === null || entitiesIngestedByTypeFromReport === null,
variables: {
input: {
query: '*',
@ -90,11 +99,13 @@ export default function IngestedAssets({ id }: Props) {
const hasSubTypeFacet = (facets || []).findIndex((facet) => facet.field === TYPE_NAMES_FACET_NAME) >= 0;
const subTypeFacets =
(hasSubTypeFacet && facets?.filter((facet) => facet.field === TYPE_NAMES_FACET_NAME)[0]) || undefined;
const countsByEntityType =
(entityTypeFacets && extractEntityTypeCountsFromFacets(entityRegistry, entityTypeFacets, subTypeFacets)) || [];
entitiesIngestedByTypeFromReport ??
(entityTypeFacets ? extractEntityTypeCountsFromFacets(entityRegistry, entityTypeFacets, subTypeFacets) : []);
// The total number of assets ingested
const total = data?.searchAcrossEntities?.total || 0;
const total = totalEntitiesIngested ?? data?.searchAcrossEntities?.total ?? 0;
return (
<>

117
datahub-web-react/src/app/ingest/source/__tests__/tests_utils.test.tsx Normal file
View File

@ -0,0 +1,117 @@
import { vi, describe, test, expect, beforeEach, afterAll } from 'vitest';
import { getEntitiesIngestedByType } from '../utils';
import { ExecutionRequestResult } from '../../../../types.generated';
// Mock the structuredReport property of ExecutionRequestResult
const mockExecutionRequestResult = (structuredReportData: any): Partial<ExecutionRequestResult> => {
return {
structuredReport: {
serializedValue: JSON.stringify(structuredReportData),
},
} as Partial<ExecutionRequestResult>;
};
describe('getEntitiesIngestedByType', () => {
// Mock for console.error
const originalConsoleError = console.error;
console.error = vi.fn();
beforeEach(() => {
vi.clearAllMocks();
});
afterAll(() => {
console.error = originalConsoleError;
});
test('returns null when structured report is not available', () => {
const result = getEntitiesIngestedByType({} as Partial<ExecutionRequestResult>);
expect(result).toBeNull();
});
test('returns null when an exception occurs during processing', () => {
// Create a malformed structured report to trigger an exception
const malformedReport = {
source: {
report: {
// Missing aspects property to trigger exception
},
},
};
const result = getEntitiesIngestedByType(mockExecutionRequestResult(malformedReport));
expect(result).toBeNull();
});
test('correctly extracts entity counts from structured report', () => {
// Create a structured report based on the example in the comments
const structuredReport = {
source: {
report: {
aspects: {
container: {
containerProperties: 156,
container: 117,
},
dataset: {
status: 1505,
schemaMetadata: 1505,
datasetProperties: 1505,
container: 1505,
operation: 1521,
},
},
},
},
};
const result = getEntitiesIngestedByType(mockExecutionRequestResult(structuredReport));
expect(result).toEqual([
{
count: 156,
displayName: 'container',
},
{
count: 1521,
displayName: 'dataset',
},
]);
});
test('handles empty aspects object', () => {
const structuredReport = {
source: {
report: {
aspects: {},
},
},
};
const result = getEntitiesIngestedByType(mockExecutionRequestResult(structuredReport));
expect(result).toEqual([]);
});
test('handles aspects with non-numeric values', () => {
const structuredReport = {
source: {
report: {
aspects: {
container: {
containerProperties: '156',
container: 117,
},
},
},
},
};
const result = getEntitiesIngestedByType(mockExecutionRequestResult(structuredReport));
expect(result).toEqual([
{
count: 156,
displayName: 'container',
},
]);
});
});

4
datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
View File

@ -193,7 +193,9 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
</StatusSection>
{(status === SUCCESS || status === SUCCEEDED_WITH_WARNINGS) && (
<IngestedAssetsSection>
{data?.executionRequest?.id && <IngestedAssets id={data?.executionRequest?.id} />}
{data?.executionRequest?.id && (
<IngestedAssets executionResult={result} id={data?.executionRequest?.id} />
)}
</IngestedAssetsSection>
)}
<LogsSection>

126
datahub-web-react/src/app/ingest/source/utils.ts
View File

@ -228,16 +228,25 @@ const transformToStructuredReport = (structuredReportObj: any): StructuredReport
}
};
export const getStructuredReport = (result: Partial<ExecutionRequestResult>): StructuredReport | null => {
// 1. Extract Serialized Structured Report
const extractStructuredReportPOJO = (result: Partial<ExecutionRequestResult>): any | null => {
const structuredReportStr = result?.structuredReport?.serializedValue;
if (!structuredReportStr) {
return null;
}
try {
return JSON.parse(structuredReportStr);
} catch (e) {
console.error(`Caught exception while parsing structured report!`, e);
return null;
}
};
// 2. Convert into JSON
const structuredReportObject = JSON.parse(structuredReportStr);
export const getStructuredReport = (result: Partial<ExecutionRequestResult>): StructuredReport | null => {
// 1. Extract Serialized Structured Report
const structuredReportObject = extractStructuredReportPOJO(result);
if (!structuredReportObject) {
return null;
}
// 3. Transform into the typed model that we have.
const structuredReport = transformToStructuredReport(structuredReportObject);
@ -246,6 +255,111 @@ export const getStructuredReport = (result: Partial<ExecutionRequestResult>): St
return structuredReport;
};
/**
* This function is used to get the total number of entities ingested from the structured report.
*
* @param result - The result of the execution request.
* @returns {number | null}
*/
export const getTotalEntitiesIngested = (result: Partial<ExecutionRequestResult>) => {
const structuredReportObject = extractStructuredReportPOJO(result);
if (!structuredReportObject) {
return null;
}
try {
return structuredReportObject.sink.report.total_records_written;
} catch (e) {
console.error(`Caught exception while parsing structured report!`, e);
return null;
}
};
/** *
* This function is used to get the entities ingested by type from the structured report.
* It returns an array of objects with the entity type and the count of entities ingested.
*
* Example input:
*
* {
* "source": {
* "report": {
* "aspects": {
* "container": {
* "containerProperties": 156,
* ...
* "container": 117
* },
* "dataset": {
* "datasetProperties": 1505,
* ...
* "operation": 1521
* },
* ...
* }
* ...
* }
* }
* ...
* }
*
* Example output:
*
* [
* {
* "count": 156,
* "displayName": "container"
* },
* ...
* ]
*
* @param result - The result of the execution request.
* @returns {EntityTypeCount[] | null}
*/
export const getEntitiesIngestedByType = (result: Partial<ExecutionRequestResult>): EntityTypeCount[] | null => {
const structuredReportObject = extractStructuredReportPOJO(result);
if (!structuredReportObject) {
return null;
}
try {
/**
* This is what the aspects object looks like in the structured report:
*
* "aspects": {
* "container": {
* "containerProperties": 156,
* ...
* "container": 117
* },
* "dataset": {
* "status": 1505,
* "schemaMetadata": 1505,
* "datasetProperties": 1505,
* "container": 1505,
* ...
* "operation": 1521
* },
* ...
* }
*/
const entities = structuredReportObject.source.report.aspects;
const entitiesIngestedByType: { [key: string]: number } = {};
Object.entries(entities).forEach(([entityName, aspects]) => {
// Get the max count of all the sub-aspects for this entity type.
entitiesIngestedByType[entityName] = Math.max(...(Object.values(aspects as object) as number[]));
});
return Object.entries(entitiesIngestedByType).map(([entityName, count]) => ({
count,
displayName: entityName,
}));
} catch (e) {
console.error(`Caught exception while parsing structured report!`, e);
return null;
}
};
export const getIngestionSourceStatus = (result?: Partial<ExecutionRequestResult> | null) => {
if (!result) {
return undefined;
@ -273,7 +387,7 @@ const ENTITIES_WITH_SUBTYPES = new Set([
EntityType.Dashboard.toLowerCase(),
]);
type EntityTypeCount = {
export type EntityTypeCount = {
count: number;
displayName: string;
};