yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-03 19:16:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: Transform Reserved keywords like quotes to OM compatible (#20459)

Browse Source
This commit is contained in:
Ayush Shah 2025-03-27 13:02:07 +05:30 committed by GitHub
parent 016d441cd0
commit 653c878497
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 79 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ingestion/examples/sample_data/looker/dashboardDataModels.json
View File

@ -41,6 +41,13 @@
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Authorize'",
"ordinalPosition": 5
},
{
"name": "5. \"Delivery\"",
"dataType": "NUMERIC",
"dataTypeDisplay": "numeric",
"description": "Vaccine Candidates in phase: 'Delivery'",
"ordinalPosition": 5
}
]
},

15
ingestion/src/metadata/ingestion/models/custom_basemodel_validation.py
View File

@ -20,22 +20,27 @@ logger = logging.getLogger("metadata")
RESTRICTED_KEYWORDS = ["::", ">"]
RESERVED_COLON_KEYWORD = "__reserved__colon__"
RESERVED_ARROW_KEYWORD = "__reserved__arrow__"
RESERVED_QUOTE_KEYWORD = "__reserved__quote__"
CREATE_ADJACENT_MODELS = {"ProfilerResponse", "SampleData"}
NAME_FIELDS = {"EntityName", "str", "ColumnName", "TableData"}
FETCH_MODELS = {"Table", "CustomColumnName"}
FETCH_MODELS = {"Table", "CustomColumnName", "DashboardDataModel"}
FIELD_NAMES = {"name", "columns", "root"}
def revert_separators(value):
return value.replace(RESERVED_COLON_KEYWORD, "::").replace(
RESERVED_ARROW_KEYWORD, ">"
return (
value.replace(RESERVED_COLON_KEYWORD, "::")
.replace(RESERVED_ARROW_KEYWORD, ">")
.replace(RESERVED_QUOTE_KEYWORD, '"')
)
def replace_separators(value):
return value.replace("::", RESERVED_COLON_KEYWORD).replace(
">", RESERVED_ARROW_KEYWORD
return (
value.replace("::", RESERVED_COLON_KEYWORD)
.replace(">", RESERVED_ARROW_KEYWORD)
.replace('"', RESERVED_QUOTE_KEYWORD)
)

60
ingestion/tests/unit/models/test_custom_pydantic.py
View File

@ -2,7 +2,14 @@ import uuid
from typing import List
from unittest import TestCase
from metadata.generated.schema.api.data.createDashboardDataModel import (
CreateDashboardDataModelRequest,
)
from metadata.generated.schema.api.data.createTable import CreateTableRequest
from metadata.generated.schema.entity.data.dashboardDataModel import (
DashboardDataModel,
DataModelType,
)
from metadata.generated.schema.entity.data.table import (
Column,
ColumnName,
@ -112,6 +119,28 @@ class CustomPydanticValidationTest(TestCase):
sourceHash=None,
)
create_request_dashboard_datamodel = CreateDashboardDataModelRequest(
name=EntityName('test"dashboarddatamodel"'),
displayName='test"dashboarddatamodel"',
description=Markdown(
root="test__reserved__quote__dashboarddatamodel__reserved__quote__"
),
dataModelType=DataModelType.PowerBIDataModel,
service=FullyQualifiedEntityName(
root='New Gyro 360.New Gyro 360."AdventureWorks2017.HumanResources"'
),
columns=[
Column(
name="struct",
dataType=DataType.STRUCT,
arrayDataType="UNKNOWN",
children=[
Column(name='test "struct_children"', dataType=DataType.BIGINT)
],
)
],
)
def test_replace_separator(self):
assert (
self.create_request.name.root
@ -126,6 +155,16 @@ class CustomPydanticValidationTest(TestCase):
== "Sales__reserved__colon__Last__reserved__arrow__Year"
)
assert (
self.create_request_dashboard_datamodel.name.root
== "test__reserved__quote__dashboarddatamodel__reserved__quote__"
)
assert (
self.create_request_dashboard_datamodel.columns[0].children[0].name.root
== "test __reserved__quote__struct_children__reserved__quote__"
)
def test_revert_separator(self):
fetch_response_revert_separator = Table(
id=uuid.uuid4(),
@ -141,6 +180,27 @@ class CustomPydanticValidationTest(TestCase):
fullyQualifiedName="test-service-table.test-db.test-schema.test",
columns=[Column(name="id", dataType=DataType.BIGINT)],
)
fetch_response_revert_separator_3 = DashboardDataModel(
id=uuid.uuid4(),
name="test__reserved__quote__dashboarddatamodel__reserved__quote__",
fullyQualifiedName="test-service-table.test-db.test-schema.test__reserved__quote__dashboarddatamodel__reserved__quote__",
dataModelType=DataModelType.PowerBIDataModel,
columns=[
Column(
name="struct",
dataType=DataType.STRUCT,
children=[
Column(name='test "struct_children"', dataType=DataType.BIGINT)
],
)
],
)
assert fetch_response_revert_separator_3.name.root == 'test"dashboarddatamodel"'
assert (
fetch_response_revert_separator_3.columns[0].children[0].name.root
== 'test "struct_children"'
)
assert fetch_response_revert_separator.name.root == "test::table"
assert fetch_response_revert_separator_2.name.root == "test::table>"

2
ingestion/tests/unit/test_fqn.py
View File

@ -59,6 +59,8 @@ class TestFqn(TestCase):
FQNTest(["a.1", "b.2", "c.3", "d"], '"a.1"."b.2"."c.3".d'),
FQNTest(["a.1", "b.2", "c.3", "d.4"], '"a.1"."b.2"."c.3"."d.4"'),
FQNTest(["fqn", "test.test.test"], 'fqn."test.test.test"'),
FQNTest(["fqn", "testtesttest"], "fqn.testtesttest"),
FQNTest(["fqn", "testtes ttest"], "fqn.testtes ttest"),
]
for x in xs:
x.validate(fqn.split(x.fqn), fqn._build(*x.parts))