fix(ui/ingest): ingestion run report (#13838)

datahub-web-react/src/app/ingestV2/source/__tests__/tests_utils.test.tsx

@@ -2,7 +2,7 @@ import { afterAll, beforeEach, describe, expect, test, vi } from 'vitest';
 
 import { SortingState } from '@components/components/Table/types';
 
-import { getEntitiesIngestedByType, getSortInput } from '@app/ingestV2/source/utils';
+import { getEntitiesIngestedByType, getSortInput, getTotalEntitiesIngested } from '@app/ingestV2/source/utils';
 
 import { ExecutionRequestResult, SortOrder } from '@types';
 
@@ -77,7 +77,7 @@ describe('getEntitiesIngestedByType', () => {
                 displayName: 'container',
             },
             {
-                count: 1521,
+                count: 1505,
                 displayName: 'dataset',
             },
         ]);
@@ -139,3 +139,102 @@ describe('getSortInput', () => {
         });
     });
 });
+
+describe('getTotalEntitiesIngested', () => {
+    test('returns null when structured report is not available', () => {
+        const result = getTotalEntitiesIngested({} as Partial<ExecutionRequestResult>);
+        expect(result).toBeNull();
+    });
+
+    test('returns null when an exception occurs during processing', () => {
+        // Create a malformed structured report to trigger an exception
+        const malformedReport = {
+            source: {
+                report: {
+                    // Missing aspects property to trigger exception
+                },
+            },
+        };
+
+        const result = getTotalEntitiesIngested(mockExecutionRequestResult(malformedReport));
+        expect(result).toBeNull();
+    });
+
+    test('returns null when aspects object is empty', () => {
+        const structuredReport = {
+            source: {
+                report: {
+                    aspects: {},
+                },
+            },
+        };
+
+        const result = getTotalEntitiesIngested(mockExecutionRequestResult(structuredReport));
+        expect(result).toBeNull();
+    });
+
+    test('correctly calculates total from multiple entity types', () => {
+        const structuredReport = {
+            source: {
+                report: {
+                    aspects: {
+                        container: {
+                            containerProperties: 156,
+                            container: 117,
+                        },
+                        dataset: {
+                            status: 1505,
+                            schemaMetadata: 1505,
+                            datasetProperties: 1505,
+                            container: 1505,
+                            operation: 1521,
+                        },
+                        dashboard: {
+                            status: 42,
+                            dashboardInfo: 42,
+                        },
+                    },
+                },
+            },
+        };
+
+        const result = getTotalEntitiesIngested(mockExecutionRequestResult(structuredReport));
+        expect(result).toBe(156 + 1505 + 42); // 1703
+    });
+
+    test('correctly calculates total from single entity type', () => {
+        const structuredReport = {
+            source: {
+                report: {
+                    aspects: {
+                        container: {
+                            containerProperties: 156,
+                            container: 117,
+                        },
+                    },
+                },
+            },
+        };
+
+        const result = getTotalEntitiesIngested(mockExecutionRequestResult(structuredReport));
+        expect(result).toBe(156);
+    });
+
+    test('handles aspects with non-numeric values', () => {
+        const structuredReport = {
+            source: {
+                report: {
+                    aspects: {
+                        container: {
+                            containerProperties: '156',
+                            container: 117,
+                        },
+                    },
+                },
+            },
+        };
+
+        const result = getTotalEntitiesIngested(mockExecutionRequestResult(structuredReport));
+        expect(result).toBe(156);
+    });
+});

datahub-web-react/src/app/ingestV2/source/utils.ts

@@ -171,26 +171,6 @@ export const getStructuredReport = (result: Partial<ExecutionRequestResult>): St
     return structuredReport;
 };
 
-/**
- * This function is used to get the total number of entities ingested from the structured report.
- *
- * @param result - The result of the execution request.
- * @returns {number | null}
- */
-export const getTotalEntitiesIngested = (result: Partial<ExecutionRequestResult>) => {
-    const structuredReportObject = extractStructuredReportPOJO(result);
-    if (!structuredReportObject) {
-        return null;
-    }
-
-    try {
-        return structuredReportObject.sink.report.total_records_written;
-    } catch (e) {
-        console.error(`Caught exception while parsing structured report!`, e);
-        return null;
-    }
-};
-
 /** *
  * This function is used to get the entities ingested by type from the structured report.
  * It returns an array of objects with the entity type and the count of entities ingested.
@@ -262,14 +242,19 @@ export const getEntitiesIngestedByType = (result: Partial<ExecutionRequestResult
         const entities = structuredReportObject.source.report.aspects;
         const entitiesIngestedByType: { [key: string]: number } = {};
         Object.entries(entities).forEach(([entityName, aspects]) => {
-            // Get the max count of all the sub-aspects for this entity type.
-            entitiesIngestedByType[entityName] = Math.max(...(Object.values(aspects as object) as number[]));
+            // Use the status aspect count instead of max count
+            const statusCount = (aspects as any)?.status;
+            if (statusCount !== undefined) {
+                entitiesIngestedByType[entityName] = statusCount;
+            } else {
+                // Get the max count of all the sub-aspects for this entity type if status is not present.
+                entitiesIngestedByType[entityName] = Math.max(...(Object.values(aspects as object) as number[]));
+            }
         });
 
         if (Object.keys(entitiesIngestedByType).length === 0) {
             return null;
         }
-
         return Object.entries(entitiesIngestedByType).map(([entityName, count]) => ({
             count,
             displayName: entityName,
@@ -280,6 +265,21 @@ export const getEntitiesIngestedByType = (result: Partial<ExecutionRequestResult
     }
 };
 
+/**
+ * This function is used to get the total number of entities ingested from the structured report.
+ *
+ * @param result - The result of the execution request.
+ * @returns {number | null}
+ */
+export const getTotalEntitiesIngested = (result: Partial<ExecutionRequestResult>) => {
+    const entityTypeCounts = getEntitiesIngestedByType(result);
+    if (!entityTypeCounts) {
+        return null;
+    }
+
+    return entityTypeCounts.reduce((total, entityType) => total + entityType.count, 0);
+};
+
 export const getIngestionSourceStatus = (result?: Partial<ExecutionRequestResult> | null) => {
     if (!result) {
         return undefined;