Feat: show dbt project name (#23044)

* Feat: show dbt project name

* Update generated TypeScript types

* added dbtSourceProject in data asset header properties

* Added tests

* Addressed comments

* Update generated TypeScript types

* move from dataAssetHeader to the dbt tab itself

* added unit test for added code

* test name change

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Ashish Gupta <ashish@getcollate.io>
(cherry picked from commit 39cb16516432ad36044c46a25f1fcbcf8732a3dc)

ingestion/src/metadata/ingestion/source/database/dbt/metadata.py

@@ -511,6 +511,7 @@ class DbtSource(DbtServiceSource):
             self.context.get().exposures = {}
             self.context.get().dbt_tests = {}
             self.context.get().run_results_generate_time = None
+
             # Since we'll be processing multiple run_results for a single project
             # we'll only consider the first run_results generated_at time
             if (
@@ -520,6 +521,9 @@ class DbtSource(DbtServiceSource):
                 self.context.get().run_results_generate_time = (
                     dbt_objects.dbt_run_results[0].metadata.generated_at
                 )
+            dbt_project_name = getattr(
+                dbt_objects.dbt_manifest.metadata, "project_name", None
+            )
             for key, manifest_node in manifest_entities.items():
                 try:
                     resource_type = getattr(
@@ -644,6 +648,7 @@ class DbtSource(DbtServiceSource):
                                     catalog_node=catalog_node,
                                 ),
                                 tags=dbt_table_tags_list or [],
+                                dbtSourceProject=dbt_project_name,
                             ),
                         )
                         yield Either(right=data_model_link)

ingestion/tests/unit/test_dbt.py

@@ -225,6 +225,7 @@ EXPECTED_DATA_MODEL_VERSIONLESS = [
     DataModel(
         modelType="DBT",
         resourceType="model",
+        dbtSourceProject="jaffle_shop",
         description="This table has basic information about a customer, as well as some derived facts based on a customer's orders",
         path="models/customers.sql",
         rawSql="sample customers raw code",
@@ -1105,6 +1106,34 @@ class DbtUnitTest(TestCase):
 
         assert len(list(filter(lambda x: x is not None, parsed_exposures))) == 0
 
+    def test_dbt_source_project_name(self):
+        """
+        Test that the DBT source project name is correctly set in the data model
+        """
+        _, dbt_objects = self.get_dbt_object_files(MOCK_SAMPLE_MANIFEST_VERSIONLESS)
+
+        # Get expected data models
+        yield_data_models = self.dbt_source_obj.yield_data_models(
+            dbt_objects=dbt_objects
+        )
+
+        for data_model_link in yield_data_models:
+            if isinstance(data_model_link, Either) and data_model_link.right:
+                data_model = data_model_link.right.datamodel
+
+                # Check that the dbtSourceProject field is correctly set
+                self.assertEqual(
+                    data_model.dbtSourceProject,
+                    "jaffle_shop",
+                    "DBT source project should be set to 'jaffle_shop'",
+                )
+
+                # Verify the field exists and is not None
+                self.assertIsNotNone(
+                    data_model.dbtSourceProject,
+                    "dbtSourceProject field should not be None",
+                )
+
     def test_constants_required_constraint_keys(self):
         """Test REQUIRED_CONSTRAINT_KEYS constant"""
         from metadata.ingestion.source.database.dbt.constants import (

openmetadata-spec/src/main/resources/json/schema/entity/data/table.json

@@ -940,6 +940,11 @@
           "description": "Resource Type of the model.",
           "type": "string"
         },
+        "dbtSourceProject": {
+          "description": "The DBT project name that served as the source for ingesting this table's metadata and lineage information.",
+          "type": "string",
+          "default": null
+        },
         "description": {
           "description": "Description of the Table from the model.",
           "$ref": "../../type/basic.json#/definitions/markdown"

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts

@@ -619,6 +619,11 @@ export interface DataModel {
      * from `schema.yaml`.
      */
     columns?: Column[];
+    /**
+     * The DBT project name that served as the source for ingesting this table's metadata and
+     * lineage information.
+     */
+    dbtSourceProject?: string;
     /**
      * Description of the Table from the model.
      */

openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts

@@ -818,6 +818,11 @@ export interface DataModel {
      * from `schema.yaml`.
      */
     columns?: Column[];
+    /**
+     * The DBT project name that served as the source for ingesting this table's metadata and
+     * lineage information.
+     */
+    dbtSourceProject?: string;
     /**
      * Description of the Table from the model.
      */

openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt-Ausführungsergebnisdateipfad",
     "dbt-run-result-http-path": "dbt-Ausführungsergebnis-HTTP-Pfad",
     "dbt-source": "dbt-Quelle",
+    "dbt-source-project": "dbt-Quellprojekt",
     "deactivated": "Deaktiviert",
     "december": "Dezember",
     "decision": "Entscheidung",

openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt Run Results File Path",
     "dbt-run-result-http-path": "dbt Run Results HTTP Path",
     "dbt-source": "dbt Source",
+    "dbt-source-project": "dbt Source Project",
     "deactivated": "Deactivated",
     "december": "December",
     "decision": "Decision",

openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "Ruta del archivo de resultados de ejecución de dbt",
     "dbt-run-result-http-path": "Ruta HTTP de resultados de ejecución de dbt",
     "dbt-source": "Fuente de dbt",
+    "dbt-source-project": "Proyecto Fuente dbt",
     "deactivated": "Desactivado",
     "december": "Diciembre",
     "decision": "Decisión",

openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "Chemin du Fichier de Résultats d'Exécution dbt",
     "dbt-run-result-http-path": "Chemin HTTP du Fichier de Résultats d'Exécution dbt",
     "dbt-source": "Source dbt",
+    "dbt-source-project": "Projet Source dbt",
     "deactivated": "Désactivé",
     "december": "Décembre",
     "decision": "Décision",

openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "Ruta do ficheiro de resultados da execución dbt",
     "dbt-run-result-http-path": "Ruta HTTP dos resultados da execución dbt",
     "dbt-source": "Fonte dbt",
+    "dbt-source-project": "Proxecto Fonte dbt",
     "deactivated": "Desactivado",
     "december": "Decembro",
     "decision": "Decisión",

openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt Run Results File Path",
     "dbt-run-result-http-path": "dbt Run Results HTTP Path",
     "dbt-source": "dbt Source",
+    "dbt-source-project": "פרויקט מקור dbt",
     "deactivated": "Deactivated",
     "december": "דצמבר",
     "decision": "החלטה",

openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt実行結果ファイルパス",
     "dbt-run-result-http-path": "dbt実行結果HTTPパス",
     "dbt-source": "dbtソース",
+    "dbt-source-project": "dbt ソースプロジェクト",
     "deactivated": "無効化済み",
     "december": "12月",
     "decision": "判断",

openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt 실행 결과 파일 경로",
     "dbt-run-result-http-path": "dbt 실행 결과 HTTP 경로",
     "dbt-source": "dbt 소스",
+    "dbt-source-project": "dbt 소스 프로젝트",
     "deactivated": "비활성화됨",
     "december": "12월",
     "decision": "결정",

openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt रन परिणाम फाइल पथ",
     "dbt-run-result-http-path": "dbt रन परिणाम HTTP पथ",
     "dbt-source": "dbt स्रोत",
+    "dbt-source-project": "dbt स्रोत प्रकल्प",
     "deactivated": "निष्क्रिय केले",
     "december": "डिसेंबर",
     "decision": "Decision",

openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt uitvoerresultaten bestandspad",
     "dbt-run-result-http-path": "dbt uitvoerresultaten HTTP-pad",
     "dbt-source": "dbt bron",
+    "dbt-source-project": "dbt Bronproject",
     "deactivated": "Gedeactiveerd",
     "december": "december",
     "decision": "Beslissing",

openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "مسیر فایل نتایج اجرای dbt",
     "dbt-run-result-http-path": "مسیر HTTP نتایج اجرای dbt",
     "dbt-source": "منبع dbt",
+    "dbt-source-project": "پروژه منبع dbt",
     "deactivated": "غیرفعال شده",
     "december": "دسامبر",
     "decision": "Decision",

openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "Caminho do Arquivo de Resultados de Execução dbt",
     "dbt-run-result-http-path": "Caminho HTTP de Resultados de Execução dbt",
     "dbt-source": "Fonte dbt",
+    "dbt-source-project": "Projeto Fonte dbt",
     "deactivated": "Desativado",
     "december": "Dezembro",
     "decision": "Decisão",

openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "Caminho do Arquivo de Resultados de Execução dbt",
     "dbt-run-result-http-path": "Caminho HTTP de Resultados de Execução dbt",
     "dbt-source": "Fonte dbt",
+    "dbt-source-project": "Projeto Fonte dbt",
     "deactivated": "Desativado",
     "december": "Dezembro",
     "decision": "Decision",

openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "Путь к файлу результатов выполнения dbt",
     "dbt-run-result-http-path": "HTTP-путь результатов запуска dbt",
     "dbt-source": "dbt Источник",
+    "dbt-source-project": "Исходный проект dbt",
     "deactivated": "Отключено",
     "december": "Декабрь",
     "decision": "Решение",

openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "เส้นทางไฟล์ผลลัพธ์การรัน dbt",
     "dbt-run-result-http-path": "เส้นทาง HTTP ของผลลัพธ์การรัน dbt",
     "dbt-source": "แหล่งที่มาของ dbt",
+    "dbt-source-project": "โปรเจกต์ต้นทาง dbt",
     "deactivated": "ถูกปิดการใช้งาน",
     "december": "ธันวาคม",
     "decision": "Decision",

openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt Çalıştırma Sonuçları Dosya Yolu",
     "dbt-run-result-http-path": "dbt Çalıştırma Sonuçları HTTP Yolu",
     "dbt-source": "dbt Kaynağı",
+    "dbt-source-project": "dbt Kaynak Projesi",
     "deactivated": "Devre Dışı Bırakıldı",
     "december": "Aralık",
     "decision": "Decision",

openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt 运行结果文件路径",
     "dbt-run-result-http-path": "dbt 运行结果 HTTP 路径",
     "dbt-source": "dbt 源",
+    "dbt-source-project": "dbt 源项目",
     "deactivated": "已停用",
     "december": "十二月",
     "decision": "Decision",

openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-tw.json

@@ -437,6 +437,7 @@
     "dbt-run-result-file-path": "dbt 執行結果檔案路徑",
     "dbt-run-result-http-path": "dbt 執行結果 HTTP 路徑",
     "dbt-source": "dbt 來源",
+    "dbt-source-project": "dbt 原始專案",
     "deactivated": "已停用",
     "december": "十二月",
     "decision": "決策",

openmetadata-ui/src/main/resources/ui/src/mocks/TableData.mock.ts

@@ -1,3 +1,4 @@
+/* eslint-disable max-len */
 /*
  *  Copyright 2022 Collate.
  *  Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,7 +12,12 @@
  *  limitations under the License.
  */
 
-import { Table } from '../generated/entity/data/table';
+import {
+  DataModel,
+  DataType,
+  ModelType,
+  Table,
+} from '../generated/entity/data/table';
 import { LabelType, State, TagSource } from '../generated/tests/testCase';
 
 export const MOCK_TABLE = {
@@ -29,7 +35,7 @@ export const MOCK_TABLE = {
   columns: [
     {
       name: 'address_id',
-      dataType: 'NUMERIC',
+      dataType: DataType.Numeric,
       dataTypeDisplay: 'numeric',
       description: 'Unique identifier for the address.',
       fullyQualifiedName:
@@ -55,7 +61,7 @@ export const MOCK_TABLE = {
     },
     {
       name: 'shop_id',
-      dataType: 'NUMERIC',
+      dataType: DataType.Numeric,
       dataTypeDisplay: 'numeric',
       description:
         'The ID of the store. This column is a foreign key reference to the shop_id column in the dim_shop table.',
@@ -82,7 +88,7 @@ export const MOCK_TABLE = {
     },
     {
       name: 'first_name',
-      dataType: 'VARCHAR',
+      dataType: DataType.Varchar,
       dataLength: 100,
       dataTypeDisplay: 'varchar',
       description: 'First name of the customer.',
@@ -106,7 +112,7 @@ export const MOCK_TABLE = {
     },
     {
       name: 'last_name',
-      dataType: 'VARCHAR',
+      dataType: DataType.Varchar,
       dataLength: 100,
       dataTypeDisplay: 'varchar',
       description: 'Last name of the customer.',
@@ -293,3 +299,38 @@ export const MOCK_TIER_DATA = {
   },
   tagFQN: 'Tier.Tier4',
 };
+
+export const MOCK_TABLE_DBT: DataModel = {
+  dbtSourceProject: 'jaffle_shop',
+  description:
+    'Staging table for payment data - cleaned and standardized raw payment information',
+  modelType: ModelType.Dbt,
+  path: 'models/staging/stg_payments.sql',
+  rawSql:
+    "with source as (\n    \n    {#-\n    Normally we would select from the table here, but we are using seeds to load\n    our data in this project\n    #}\n    select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n    select\n        id as payment_id,\n        order_id,\n        payment_method,\n        cycle_name,\n\n        -- `amount` is currently stored in cents, so we convert it to dollars\n        amount / 100 as amount\n\n    from source\n\n)\n\nselect * from renamed",
+  resourceType: 'model',
+  sql: 'with source as (\n    select * from "dev"."dbt_production"."raw_payments"\n\n),\n\nrenamed as (\n\n    select\n        id as payment_id,\n        order_id,\n        payment_method,\n        cycle_name,\n\n        -- `amount` is currently stored in cents, so we convert it to dollars\n        amount / 100 as amount\n\n    from source\n\n)\n\nselect * from renamed',
+  tags: [],
+  upstream: ['redshift.dev.dbt_production.raw_payments'],
+  columns: [
+    {
+      children: [],
+      tags: [],
+      dataLength: 1,
+      dataType: DataType.Int,
+      description: 'Unique identifier for each payment transaction',
+      name: 'payment_id',
+      ordinalPosition: 1,
+    },
+    {
+      children: [],
+      tags: [],
+      dataLength: 1,
+      dataType: DataType.Varchar,
+      description:
+        'Method used for payment (credit_card, coupon, bank_transfer, gift_card)',
+      name: '"payment_method"',
+      ordinalPosition: 3,
+    },
+  ],
+};

openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.test.tsx

@@ -11,8 +11,10 @@
  *  limitations under the License.
  */
 import { OperationPermission } from '../context/PermissionProvider/PermissionProvider.interface';
+import { EntityTabs } from '../enums/entity.enum';
 import { TagLabel } from '../generated/entity/data/container';
 import { Column, DataType } from '../generated/entity/data/table';
+import { MOCK_TABLE, MOCK_TABLE_DBT } from '../mocks/TableData.mock';
 import {
   ExtraTableDropdownOptions,
   findColumnByEntityLink,
@@ -21,6 +23,7 @@ import {
   getSafeExpandAllKeys,
   getSchemaDepth,
   getSchemaFieldCount,
+  getTableDetailPageBaseTabs,
   getTagsWithoutTier,
   getTierTags,
   isLargeSchema,
@@ -913,4 +916,85 @@ describe('TableUtils', () => {
       });
     });
   });
+
+  const mockProps = {
+    activeTab: EntityTabs.DBT,
+    deleted: false,
+    editCustomAttributePermission: true,
+    editLineagePermission: true,
+    feedCount: {
+      closedTaskCount: 0,
+      conversationCount: 0,
+      mentionCount: 0,
+      openTaskCount: 0,
+      totalCount: 0,
+      totalTasksCount: 0,
+    },
+    fetchTableDetails: jest.fn(),
+    getEntityFeedCount: jest.fn(),
+    handleFeedCount: jest.fn(),
+    isTourOpen: false,
+    isViewTableType: true,
+    queryCount: 0,
+    viewAllPermission: true,
+    viewQueriesPermission: true,
+    viewSampleDataPermission: true,
+    viewProfilerPermission: true,
+    tablePermissions: {
+      Create: true,
+      Delete: true,
+      EditAll: true,
+      EditCertification: true,
+      EditCustomFields: true,
+      EditDataProfile: true,
+      EditDescription: true,
+      EditDisplayName: true,
+      EditEntityRelationship: true,
+      EditGlossaryTerms: true,
+      EditLineage: true,
+      EditOwners: true,
+      EditQueries: true,
+      EditSampleData: true,
+      EditTags: true,
+      EditTests: true,
+      EditTier: true,
+      ViewAll: true,
+      ViewBasic: true,
+      ViewDataProfile: true,
+      ViewProfilerGlobalConfiguration: true,
+      ViewQueries: true,
+      ViewSampleData: true,
+      ViewTests: true,
+      ViewUsage: true,
+    } as OperationPermission,
+    tableDetails: { ...MOCK_TABLE, dataModel: MOCK_TABLE_DBT },
+  };
+
+  describe('TableDetailPage Tabs', () => {
+    it('dbt tab should render dbtSourceProject with value', () => {
+      const result = getTableDetailPageBaseTabs(mockProps);
+      const stringifyResult = JSON.stringify(result[7].children);
+
+      expect(stringifyResult).toContain('label.dbt-source-project:');
+      expect(stringifyResult).toContain(
+        '{"data-testid":"dbt-source-project-id","children":"jaffle_shop"}'
+      );
+    });
+
+    it('dbt tab should render dbtSourceProject with value No data placeholder', () => {
+      const result = getTableDetailPageBaseTabs({
+        ...mockProps,
+        tableDetails: {
+          ...MOCK_TABLE,
+          dataModel: { ...MOCK_TABLE_DBT, dbtSourceProject: undefined },
+        },
+      });
+      const stringifyResult = JSON.stringify(result[7].children);
+
+      expect(stringifyResult).toContain('label.dbt-source-project:');
+      expect(stringifyResult).toContain(
+        '{"data-testid":"dbt-source-project-id","children":"--"}'
+      );
+    });
+  });
 });

openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx

@@ -12,7 +12,7 @@
  */
 
 import Icon, { SearchOutlined } from '@ant-design/icons';
-import { Space, Tooltip, Typography } from 'antd';
+import { Divider, Space, Tooltip, Typography } from 'antd';
 import { ExpandableConfig } from 'antd/lib/table/interface';
 import classNames from 'classnames';
 import {
@@ -138,7 +138,7 @@ import Lineage from '../components/Lineage/Lineage.component';
 import { SourceType } from '../components/SearchedData/SearchedData.interface';
 import { NON_SERVICE_TYPE_ASSETS } from '../constants/Assets.constants';
 import { FQN_SEPARATOR_CHAR } from '../constants/char.constants';
-import { DE_ACTIVE_COLOR } from '../constants/constants';
+import { DE_ACTIVE_COLOR, NO_DATA_PLACEHOLDER } from '../constants/constants';
 import { ExportTypes } from '../constants/Export.constants';
 import LineageProvider from '../context/LineageProvider/LineageProvider';
 import { OperationPermission } from '../context/PermissionProvider/PermissionProvider.interface';
@@ -924,11 +924,30 @@ export const getTableDetailPageBaseTabs = ({
             get(tableDetails, 'dataModel.rawSql', '')
           }
           title={
-            <Space className="p-y-xss">
+            <Space className="p-y-xss" size="small">
-              <Typography.Text className="text-grey-muted">
+              <div>
-                {`${t('label.path')}:`}
+                <Typography.Text className="text-grey-muted">
-              </Typography.Text>
+                  {`${t('label.dbt-source-project')}: `}
-              <Typography.Text>{tableDetails?.dataModel?.path}</Typography.Text>
+                </Typography.Text>
+                <Typography.Text data-testid="dbt-source-project-id">
+                  {tableDetails?.dataModel?.dbtSourceProject ??
+                    NO_DATA_PLACEHOLDER}
+                </Typography.Text>
+              </div>
+
+              <Divider
+                className="self-center vertical-divider"
+                type="vertical"
+              />
+
+              <div>
+                <Typography.Text className="text-grey-muted">
+                  {`${t('label.path')}: `}
+                </Typography.Text>
+                <Typography.Text>
+                  {tableDetails?.dataModel?.path}
+                </Typography.Text>
+              </div>
             </Space>
           }
         />