yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-17 10:35:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #16235: need quote fullyQualifiedName in Ingestion Framework (#16273)

Browse Source 
* Fixes #16235: need quote fullyQualifiedName in Ingestion Framework

* MINOR: fix UT issue

* revert: fix UT issue

* revert code

* revert code

* format code
This commit is contained in:
juntao 2024-05-23 23:45:47 +08:00 committed by GitHub
parent 015d71c9de
commit 8dd613caa5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 19 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingestion/src/metadata/data_insight/processor/reports/web_analytic_report_data_processor.py
View File

@ -108,7 +108,9 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor):
entity_obj = EntityObj(split_url[0], split_url[1]) entity_obj = EntityObj(split_url[0], split_url[1])
entity_type = entity_obj.entity_type entity_type = entity_obj.entity_type
re_pattern = re.compile(f"(.*{entity_type}/{entity_obj.fqn})") re_pattern = re.compile(
f"(.*{re.escape(entity_type)}/{re.escape(entity_obj.fqn)})"
)
if ( if (
entity_obj.fqn in refined_data entity_obj.fqn in refined_data

2
ingestion/src/metadata/ingestion/ometa/mixins/table_mixin.py
View File

@ -290,7 +290,7 @@ class OMetaTableMixin:
Returns: Returns:
Optional[Table]: OM table object Optional[Table]: OM table object
""" """
return self._get(Table, f"{quote(model_str(fqn))}/tableProfile/latest") return self._get(Table, f"{quote(model_str(fqn), safe='')}/tableProfile/latest")
def create_or_update_custom_metric( def create_or_update_custom_metric(
self, custom_metric: CreateCustomMetricRequest, table_id: str self, custom_metric: CreateCustomMetricRequest, table_id: str

13
ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py
View File

@ -89,8 +89,9 @@ class User(Base):
signedup = Column(DateTime) signedup = Column(DateTime)
# with weird characters of fqn
class NewUser(Base): class NewUser(Base):
__tablename__ = "new_users" __tablename__ = "new/users"
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
name = Column(String(256)) name = Column(String(256))
fullname = Column(String(256)) fullname = Column(String(256))
@ -303,7 +304,7 @@ class ProfilerWorkflowTest(TestCase):
{ {
"type": "Profiler", "type": "Profiler",
"profileSample": 50, "profileSample": 50,
"tableFilterPattern": {"includes": ["new_users"]}, "tableFilterPattern": {"includes": ["new/users"]},
} }
) )
workflow_config["processor"] = {"type": "orm-profiler", "config": {}} workflow_config["processor"] = {"type": "orm-profiler", "config": {}}
@ -315,13 +316,19 @@ class ProfilerWorkflowTest(TestCase):
table = self.metadata.get_by_name( table = self.metadata.get_by_name(
entity=Table, entity=Table,
fqn="test_sqlite.main.main.new_users", fqn="test_sqlite.main.main.new/users",
fields=["tableProfilerConfig"], fields=["tableProfilerConfig"],
) )
# setting sampleProfile from config has been temporarly removed # setting sampleProfile from config has been temporarly removed
# up until we split tests and profiling # up until we split tests and profiling
assert table.tableProfilerConfig is None assert table.tableProfilerConfig is None
profile = self.metadata.get_latest_table_profile(
table.fullyQualifiedName
).profile
assert profile is not None
def test_workflow_datetime_partition(self): def test_workflow_datetime_partition(self):
"""test workflow with partition""" """test workflow with partition"""
workflow_config = deepcopy(ingestion_config) workflow_config = deepcopy(ingestion_config)

10
ingestion/tests/unit/data_insight/test_web_analytic_report_processor.py
View File

@ -42,8 +42,8 @@ WEB_ANALYTIC_EVENTS = [
timestamp=1667475458757, timestamp=1667475458757,
eventType=WebAnalyticEventType.PageView.value, eventType=WebAnalyticEventType.PageView.value,
eventData=PageViewData( eventData=PageViewData(
fullUrl="http://localhost:8585/table/sample_data.ecommerce_db.shopify.dim_address", fullUrl="http://localhost:8585/table/sample_data.ecommerce_db.shopify.dim(address)",
url="/table/sample_data.ecommerce_db.shopify.dim_address", url="/table/sample_data.ecommerce_db.shopify.dim(address)",
hostname="localhost", hostname="localhost",
language="en-US", language="en-US",
screenSize="2140x1273", screenSize="2140x1273",
@ -58,8 +58,8 @@ WEB_ANALYTIC_EVENTS = [
timestamp=1667475458757, timestamp=1667475458757,
eventType=WebAnalyticEventType.PageView.value, eventType=WebAnalyticEventType.PageView.value,
eventData=PageViewData( eventData=PageViewData(
fullUrl="http://localhost:8585/table/sample_data.ecommerce_db.shopify.dim_address", fullUrl="http://localhost:8585/table/sample_data.ecommerce_db.shopify.dim(address)",
url="/table/sample_data.ecommerce_db.shopify.dim_address", url="/table/sample_data.ecommerce_db.shopify.dim(address)",
hostname="localhost", hostname="localhost",
language="en-US", language="en-US",
screenSize="2140x1273", screenSize="2140x1273",
@ -111,7 +111,7 @@ class WebAnalyticEntityViewReportDataProcessorTest(unittest.TestCase):
assert ( assert (
web_analytic_entity_report_data[ web_analytic_entity_report_data[
"sample_data.ecommerce_db.shopify.dim_address" "sample_data.ecommerce_db.shopify.dim(address)"
].views ].views
== 2 == 2
) )