yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-19 04:41:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat: show dbt project name (#23044)

Browse Source 
* Feat: show dbt project name

* Update generated TypeScript types

* added dbtSourceProject in data asset header properties

* Added tests

* Addressed comments

* Update generated TypeScript types

* move from dataAssetHeader to the dbt tab itself

* added unit test for added code

* test name change

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Ashish Gupta <ashish@getcollate.io>
(cherry picked from commit 39cb16516432ad36044c46a25f1fcbcf8732a3dc)
This commit is contained in:
Suman Maharana 2025-09-10 14:53:28 +05:30 committed by OpenMetadata Release Bot
parent 381c1ce745
commit f76f2be8f7
26 changed files with 223 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ingestion/src/metadata/ingestion/source/database/dbt/metadata.py
View File

@ -511,6 +511,7 @@ class DbtSource(DbtServiceSource):
self.context.get().exposures = {}
self.context.get().dbt_tests = {}
self.context.get().run_results_generate_time = None
# Since we'll be processing multiple run_results for a single project
# we'll only consider the first run_results generated_at time
if (
@ -520,6 +521,9 @@ class DbtSource(DbtServiceSource):
self.context.get().run_results_generate_time = (
dbt_objects.dbt_run_results[0].metadata.generated_at
)
dbt_project_name = getattr(
dbt_objects.dbt_manifest.metadata, "project_name", None
)
for key, manifest_node in manifest_entities.items():
try:
resource_type = getattr(
@ -644,6 +648,7 @@ class DbtSource(DbtServiceSource):
catalog_node=catalog_node,
),
tags=dbt_table_tags_list or [],
dbtSourceProject=dbt_project_name,
),
)
yield Either(right=data_model_link)

29
ingestion/tests/unit/test_dbt.py
View File

@ -225,6 +225,7 @@ EXPECTED_DATA_MODEL_VERSIONLESS = [
DataModel(
modelType="DBT",
resourceType="model",
dbtSourceProject="jaffle_shop",
description="This table has basic information about a customer, as well as some derived facts based on a customer's orders",
path="models/customers.sql",
rawSql="sample customers raw code",
@ -1105,6 +1106,34 @@ class DbtUnitTest(TestCase):
assert len(list(filter(lambda x: x is not None, parsed_exposures))) == 0
def test_dbt_source_project_name(self):
"""
Test that the DBT source project name is correctly set in the data model
"""
_, dbt_objects = self.get_dbt_object_files(MOCK_SAMPLE_MANIFEST_VERSIONLESS)
# Get expected data models
yield_data_models = self.dbt_source_obj.yield_data_models(
dbt_objects=dbt_objects
)
for data_model_link in yield_data_models:
if isinstance(data_model_link, Either) and data_model_link.right:
data_model = data_model_link.right.datamodel
# Check that the dbtSourceProject field is correctly set
self.assertEqual(
data_model.dbtSourceProject,
"jaffle_shop",
"DBT source project should be set to 'jaffle_shop'",
)
# Verify the field exists and is not None
self.assertIsNotNone(
data_model.dbtSourceProject,
"dbtSourceProject field should not be None",
)
def test_constants_required_constraint_keys(self):
"""Test REQUIRED_CONSTRAINT_KEYS constant"""
from metadata.ingestion.source.database.dbt.constants import (

5
openmetadata-spec/src/main/resources/json/schema/entity/data/table.json
View File

@ -940,6 +940,11 @@
"description": "Resource Type of the model.",
"type": "string"
},
"dbtSourceProject": {
"description": "The DBT project name that served as the source for ingesting this table's metadata and lineage information.",
"type": "string",
"default": null
},
"description": {
"description": "Description of the Table from the model.",
"$ref": "../../type/basic.json#/definitions/markdown"

5
openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts
View File

@ -619,6 +619,11 @@ export interface DataModel {
* from `schema.yaml`.
*/
columns?: Column[];
/**
* The DBT project name that served as the source for ingesting this table's metadata and
* lineage information.
*/
dbtSourceProject?: string;
/**
* Description of the Table from the model.
*/

5
openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts
View File

@ -818,6 +818,11 @@ export interface DataModel {
* from `schema.yaml`.
*/
columns?: Column[];
/**
* The DBT project name that served as the source for ingesting this table's metadata and
* lineage information.
*/
dbtSourceProject?: string;
/**
* Description of the Table from the model.
*/

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt-Ausführungsergebnisdateipfad",
"dbt-run-result-http-path": "dbt-Ausführungsergebnis-HTTP-Pfad",
"dbt-source": "dbt-Quelle",
"dbt-source-project": "dbt-Quellprojekt",
"deactivated": "Deaktiviert",
"december": "Dezember",
"decision": "Entscheidung",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt Run Results File Path",
"dbt-run-result-http-path": "dbt Run Results HTTP Path",
"dbt-source": "dbt Source",
"dbt-source-project": "dbt Source Project",
"deactivated": "Deactivated",
"december": "December",
"decision": "Decision",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "Ruta del archivo de resultados de ejecución de dbt",
"dbt-run-result-http-path": "Ruta HTTP de resultados de ejecución de dbt",
"dbt-source": "Fuente de dbt",
"dbt-source-project": "Proyecto Fuente dbt",
"deactivated": "Desactivado",
"december": "Diciembre",
"decision": "Decisión",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "Chemin du Fichier de Résultats d'Exécution dbt",
"dbt-run-result-http-path": "Chemin HTTP du Fichier de Résultats d'Exécution dbt",
"dbt-source": "Source dbt",
"dbt-source-project": "Projet Source dbt",
"deactivated": "Désactivé",
"december": "Décembre",
"decision": "Décision",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "Ruta do ficheiro de resultados da execución dbt",
"dbt-run-result-http-path": "Ruta HTTP dos resultados da execución dbt",
"dbt-source": "Fonte dbt",
"dbt-source-project": "Proxecto Fonte dbt",
"deactivated": "Desactivado",
"december": "Decembro",
"decision": "Decisión",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt Run Results File Path",
"dbt-run-result-http-path": "dbt Run Results HTTP Path",
"dbt-source": "dbt Source",
"dbt-source-project": "פרויקט מקור dbt",
"deactivated": "Deactivated",
"december": "דצמבר",
"decision": "החלטה",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt実行結果ファイルパス",
"dbt-run-result-http-path": "dbt実行結果HTTPパス",
"dbt-source": "dbtソース",
"dbt-source-project": "dbt ソースプロジェクト",
"deactivated": "無効化済み",
"december": "12月",
"decision": "判断",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/ko-kr.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt 실행 결과 파일 경로",
"dbt-run-result-http-path": "dbt 실행 결과 HTTP 경로",
"dbt-source": "dbt 소스",
"dbt-source-project": "dbt 소스 프로젝트",
"deactivated": "비활성화됨",
"december": "12월",
"decision": "결정",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/mr-in.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt रन परिणाम फाइल पथ",
"dbt-run-result-http-path": "dbt रन परिणाम HTTP पथ",
"dbt-source": "dbt स्रोत",
"dbt-source-project": "dbt स्रोत प्रकल्प",
"deactivated": "निष्क्रिय केले",
"december": "डिसेंबर",
"decision": "Decision",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt uitvoerresultaten bestandspad",
"dbt-run-result-http-path": "dbt uitvoerresultaten HTTP-pad",
"dbt-source": "dbt bron",
"dbt-source-project": "dbt Bronproject",
"deactivated": "Gedeactiveerd",
"december": "december",
"decision": "Beslissing",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "مسیر فایل نتایج اجرای dbt",
"dbt-run-result-http-path": "مسیر HTTP نتایج اجرای dbt",
"dbt-source": "منبع dbt",
"dbt-source-project": "پروژه منبع dbt",
"deactivated": "غیرفعال شده",
"december": "دسامبر",
"decision": "Decision",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "Caminho do Arquivo de Resultados de Execução dbt",
"dbt-run-result-http-path": "Caminho HTTP de Resultados de Execução dbt",
"dbt-source": "Fonte dbt",
"dbt-source-project": "Projeto Fonte dbt",
"deactivated": "Desativado",
"december": "Dezembro",
"decision": "Decisão",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-pt.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "Caminho do Arquivo de Resultados de Execução dbt",
"dbt-run-result-http-path": "Caminho HTTP de Resultados de Execução dbt",
"dbt-source": "Fonte dbt",
"dbt-source-project": "Projeto Fonte dbt",
"deactivated": "Desativado",
"december": "Dezembro",
"decision": "Decision",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "Путь к файлу результатов выполнения dbt",
"dbt-run-result-http-path": "HTTP-путь результатов запуска dbt",
"dbt-source": "dbt Источник",
"dbt-source-project": "Исходный проект dbt",
"deactivated": "Отключено",
"december": "Декабрь",
"decision": "Решение",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/th-th.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "เส้นทางไฟล์ผลลัพธ์การรัน dbt",
"dbt-run-result-http-path": "เส้นทาง HTTP ของผลลัพธ์การรัน dbt",
"dbt-source": "แหล่งที่มาของ dbt",
"dbt-source-project": "โปรเจกต์ต้นทาง dbt",
"deactivated": "ถูกปิดการใช้งาน",
"december": "ธันวาคม",
"decision": "Decision",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/tr-tr.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt Çalıştırma Sonuçları Dosya Yolu",
"dbt-run-result-http-path": "dbt Çalıştırma Sonuçları HTTP Yolu",
"dbt-source": "dbt Kaynağı",
"dbt-source-project": "dbt Kaynak Projesi",
"deactivated": "Devre Dışı Bırakıldı",
"december": "Aralık",
"decision": "Decision",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt 运行结果文件路径",
"dbt-run-result-http-path": "dbt 运行结果 HTTP 路径",
"dbt-source": "dbt 源",
"dbt-source-project": "dbt 源项目",
"deactivated": "已停用",
"december": "十二月",
"decision": "Decision",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-tw.json
View File

@ -437,6 +437,7 @@
"dbt-run-result-file-path": "dbt 執行結果檔案路徑",
"dbt-run-result-http-path": "dbt 執行結果 HTTP 路徑",
"dbt-source": "dbt 來源",
"dbt-source-project": "dbt 原始專案",
"deactivated": "已停用",
"december": "十二月",
"decision": "決策",

51
openmetadata-ui/src/main/resources/ui/src/mocks/TableData.mock.ts
View File

@ -1,3 +1,4 @@
/* eslint-disable max-len */
/*
* Copyright 2022 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
@ -11,7 +12,12 @@
* limitations under the License.
*/
import { Table } from '../generated/entity/data/table';
import {
DataModel,
DataType,
ModelType,
Table,
} from '../generated/entity/data/table';
import { LabelType, State, TagSource } from '../generated/tests/testCase';
export const MOCK_TABLE = {
@ -29,7 +35,7 @@ export const MOCK_TABLE = {
columns: [
{
name: 'address_id',
dataType: 'NUMERIC',
dataType: DataType.Numeric,
dataTypeDisplay: 'numeric',
description: 'Unique identifier for the address.',
fullyQualifiedName:
@ -55,7 +61,7 @@ export const MOCK_TABLE = {
},
{
name: 'shop_id',
dataType: 'NUMERIC',
dataType: DataType.Numeric,
dataTypeDisplay: 'numeric',
description:
'The ID of the store. This column is a foreign key reference to the shop_id column in the dim_shop table.',
@ -82,7 +88,7 @@ export const MOCK_TABLE = {
},
{
name: 'first_name',
dataType: 'VARCHAR',
dataType: DataType.Varchar,
dataLength: 100,
dataTypeDisplay: 'varchar',
description: 'First name of the customer.',
@ -106,7 +112,7 @@ export const MOCK_TABLE = {
},
{
name: 'last_name',
dataType: 'VARCHAR',
dataType: DataType.Varchar,
dataLength: 100,
dataTypeDisplay: 'varchar',
description: 'Last name of the customer.',
@ -293,3 +299,38 @@ export const MOCK_TIER_DATA = {
},
tagFQN: 'Tier.Tier4',
};
export const MOCK_TABLE_DBT: DataModel = {
dbtSourceProject: 'jaffle_shop',
description:
'Staging table for payment data - cleaned and standardized raw payment information',
modelType: ModelType.Dbt,
path: 'models/staging/stg_payments.sql',
rawSql:
"with source as (\n \n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n cycle_name,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed",
resourceType: 'model',
sql: 'with source as (\n select * from "dev"."dbt_production"."raw_payments"\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n cycle_name,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed',
tags: [],
upstream: ['redshift.dev.dbt_production.raw_payments'],
columns: [
{
children: [],
tags: [],
dataLength: 1,
dataType: DataType.Int,
description: 'Unique identifier for each payment transaction',
name: 'payment_id',
ordinalPosition: 1,
},
{
children: [],
tags: [],
dataLength: 1,
dataType: DataType.Varchar,
description:
'Method used for payment (credit_card, coupon, bank_transfer, gift_card)',
name: '"payment_method"',
ordinalPosition: 3,
},
],
};

84
openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.test.tsx
View File

@ -11,8 +11,10 @@
* limitations under the License.
*/
import { OperationPermission } from '../context/PermissionProvider/PermissionProvider.interface';
import { EntityTabs } from '../enums/entity.enum';
import { TagLabel } from '../generated/entity/data/container';
import { Column, DataType } from '../generated/entity/data/table';
import { MOCK_TABLE, MOCK_TABLE_DBT } from '../mocks/TableData.mock';
import {
ExtraTableDropdownOptions,
findColumnByEntityLink,
@ -21,6 +23,7 @@ import {
getSafeExpandAllKeys,
getSchemaDepth,
getSchemaFieldCount,
getTableDetailPageBaseTabs,
getTagsWithoutTier,
getTierTags,
isLargeSchema,
@ -913,4 +916,85 @@ describe('TableUtils', () => {
});
});
});
const mockProps = {
activeTab: EntityTabs.DBT,
deleted: false,
editCustomAttributePermission: true,
editLineagePermission: true,
feedCount: {
closedTaskCount: 0,
conversationCount: 0,
mentionCount: 0,
openTaskCount: 0,
totalCount: 0,
totalTasksCount: 0,
},
fetchTableDetails: jest.fn(),
getEntityFeedCount: jest.fn(),
handleFeedCount: jest.fn(),
isTourOpen: false,
isViewTableType: true,
queryCount: 0,
viewAllPermission: true,
viewQueriesPermission: true,
viewSampleDataPermission: true,
viewProfilerPermission: true,
tablePermissions: {
Create: true,
Delete: true,
EditAll: true,
EditCertification: true,
EditCustomFields: true,
EditDataProfile: true,
EditDescription: true,
EditDisplayName: true,
EditEntityRelationship: true,
EditGlossaryTerms: true,
EditLineage: true,
EditOwners: true,
EditQueries: true,
EditSampleData: true,
EditTags: true,
EditTests: true,
EditTier: true,
ViewAll: true,
ViewBasic: true,
ViewDataProfile: true,
ViewProfilerGlobalConfiguration: true,
ViewQueries: true,
ViewSampleData: true,
ViewTests: true,
ViewUsage: true,
} as OperationPermission,
tableDetails: { ...MOCK_TABLE, dataModel: MOCK_TABLE_DBT },
};
describe('TableDetailPage Tabs', () => {
it('dbt tab should render dbtSourceProject with value', () => {
const result = getTableDetailPageBaseTabs(mockProps);
const stringifyResult = JSON.stringify(result[7].children);
expect(stringifyResult).toContain('label.dbt-source-project:');
expect(stringifyResult).toContain(
'{"data-testid":"dbt-source-project-id","children":"jaffle_shop"}'
);
});
it('dbt tab should render dbtSourceProject with value No data placeholder', () => {
const result = getTableDetailPageBaseTabs({
...mockProps,
tableDetails: {
...MOCK_TABLE,
dataModel: { ...MOCK_TABLE_DBT, dbtSourceProject: undefined },
},
});
const stringifyResult = JSON.stringify(result[7].children);
expect(stringifyResult).toContain('label.dbt-source-project:');
expect(stringifyResult).toContain(
'{"data-testid":"dbt-source-project-id","children":"--"}'
);
});
});
});

27
openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx
View File

@ -12,7 +12,7 @@
*/
import Icon, { SearchOutlined } from '@ant-design/icons';
import { Space, Tooltip, Typography } from 'antd';
import { Divider, Space, Tooltip, Typography } from 'antd';
import { ExpandableConfig } from 'antd/lib/table/interface';
import classNames from 'classnames';
import {
@ -138,7 +138,7 @@ import Lineage from '../components/Lineage/Lineage.component';
import { SourceType } from '../components/SearchedData/SearchedData.interface';
import { NON_SERVICE_TYPE_ASSETS } from '../constants/Assets.constants';
import { FQN_SEPARATOR_CHAR } from '../constants/char.constants';
import { DE_ACTIVE_COLOR } from '../constants/constants';
import { DE_ACTIVE_COLOR, NO_DATA_PLACEHOLDER } from '../constants/constants';
import { ExportTypes } from '../constants/Export.constants';
import LineageProvider from '../context/LineageProvider/LineageProvider';
import { OperationPermission } from '../context/PermissionProvider/PermissionProvider.interface';
@ -924,11 +924,30 @@ export const getTableDetailPageBaseTabs = ({
get(tableDetails, 'dataModel.rawSql', '')
}
title={
<Space className="p-y-xss">
<Space className="p-y-xss" size="small">
<div>
<Typography.Text className="text-grey-muted">
{`${t('label.dbt-source-project')}: `}
</Typography.Text>
<Typography.Text data-testid="dbt-source-project-id">
{tableDetails?.dataModel?.dbtSourceProject ??
NO_DATA_PLACEHOLDER}
</Typography.Text>
</div>
<Divider
className="self-center vertical-divider"
type="vertical"
/>
<div>
<Typography.Text className="text-grey-muted">
{`${t('label.path')}: `}
</Typography.Text>
<Typography.Text>{tableDetails?.dataModel?.path}</Typography.Text>
<Typography.Text>
{tableDetails?.dataModel?.path}
</Typography.Text>
</div>
</Space>
}
/>