yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-12 18:47:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ingestion): remove company domain for security reason (#10839)

Browse Source
This commit is contained in:
Shubham Jagtap 2024-07-09 09:45:20 +05:30 committed by GitHub
parent 8722878b69
commit b6c7fe8267
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 86 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json
View File

@ -390,7 +390,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
@ -1709,7 +1709,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_cll.json
View File

@ -483,7 +483,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

4
metadata-ingestion/tests/integration/powerbi/golden_test_container.json
View File

@ -780,7 +780,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
@ -2213,7 +2213,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json
View File

@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json
View File

@ -482,7 +482,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json
View File

@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json
View File

@ -465,7 +465,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json
View File

@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json
View File

@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

4
metadata-ingestion/tests/integration/powerbi/golden_test_report.json
View File

@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},
@ -1493,7 +1493,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json
View File

@ -382,7 +382,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json
View File

@ -490,7 +490,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1",
"viewLanguage": "m_query"
}
},

2
metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py
View File

@ -283,7 +283,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None
"name": "job-history",
"source": [
{
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
}
],
"datasourceUsages": [

2
metadata-ingestion/tests/integration/powerbi/test_m_parser.py
View File

@ -35,7 +35,7 @@ M_QUERIES = [
'let\n Source = Sql.Database("AUPRDWHDB", "COMMOPSDB", [Query="Select#(lf)*,#(lf)concat((UPPER(REPLACE(SALES_SPECIALIST,\'-\',\'\'))),#(lf)LEFT(CAST(INVOICE_DATE AS DATE),4)+LEFT(RIGHT(CAST(INVOICE_DATE AS DATE),5),2)) AS AGENT_KEY,#(lf)CASE#(lf) WHEN CLASS = \'Software\' and (NOT(PRODUCT in (\'ADV\', \'Adv\') and left(ACCOUNT_ID,2)=\'10\') #(lf) or V_ENTERPRISE_INVOICED_REVENUE.TYPE = \'Manual Adjustment\') THEN INVOICE_AMOUNT#(lf) WHEN V_ENTERPRISE_INVOICED_REVENUE.TYPE IN (\'Recurring\',\'0\') THEN INVOICE_AMOUNT#(lf) ELSE 0#(lf)END as SOFTWARE_INV#(lf)#(lf)from V_ENTERPRISE_INVOICED_REVENUE", CommandTimeout=#duration(0, 1, 30, 0)]),\n #"Added Conditional Column" = Table.AddColumn(Source, "Services", each if [CLASS] = "Services" then [INVOICE_AMOUNT] else 0),\n #"Added Custom" = Table.AddColumn(#"Added Conditional Column", "Advanced New Sites", each if [PRODUCT] = "ADV"\nor [PRODUCT] = "Adv"\nthen [NEW_SITE]\nelse 0)\nin\n #"Added Custom"',
'let\n Source = Snowflake.Databases("ghh48144.snowflakefakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]),\n Source2 = PostgreSQL.Database("localhost", "mics"),\n public_order_date = Source2{[Schema="public",Item="order_date"]}[Data],\n GSL_TEST_DB_Database = Source{[Name="GSL_TEST_DB",Kind="Database"]}[Data],\n PUBLIC_Schema = GSL_TEST_DB_Database{[Name="PUBLIC",Kind="Schema"]}[Data],\n SALES_ANALYST_VIEW_View = PUBLIC_Schema{[Name="SALES_ANALYST_VIEW",Kind="View"]}[Data],\n two_source_table = Table.Combine({public_order_date, SALES_ANALYST_VIEW_View})\n in\n two_source_table',
'let\n Source = PostgreSQL.Database("localhost" , "mics" ),\n public_order_date = Source{[Schema="public",Item="order_date"]}[Data] \n in \n public_order_date',
'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
'let\n Source = Sql.Database("localhost", "library"),\n dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]\n in dbo_book_issue',
'let\n Source = Snowflake.Databases("ghh48144.snowflakefakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]),\n GSL_TEST_DB_Database = Source{[Name="GSL_TEST_DB",Kind="Database"]}[Data],\n PUBLIC_Schema = GSL_TEST_DB_Database{[Name="PUBLIC",Kind="Schema"]}[Data],\n SALES_FORECAST_Table = PUBLIC_Schema{[Name="SALES_FORECAST",Kind="Table"]}[Data],\n SALES_ANALYST_Table = PUBLIC_Schema{[Name="SALES_ANALYST",Kind="Table"]}[Data],\n RESULT = Table.Combine({SALES_FORECAST_Table, SALES_ANALYST_Table})\n\nin\n RESULT',
'let\n Source = GoogleBigQuery.Database(),\n #"seraphic-music-344307" = Source{[Name="seraphic-music-344307"]}[Data],\n school_dataset_Schema = #"seraphic-music-344307"{[Name="school_dataset",Kind="Schema"]}[Data],\n first_Table = school_dataset_Schema{[Name="first",Kind="Table"]}[Data]\nin\n first_Table',

2
metadata-ingestion/tests/integration/powerbi/test_powerbi.py
View File

@ -393,7 +393,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -
"name": "job-history",
"source": [
{
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
}
],
"datasourceUsages": [

25
metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json
View File

@ -89,10 +89,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -100,7 +101,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327546106,
"lastObserved": 1720073671135,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
@ -319,10 +320,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -330,7 +332,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327546129,
"lastObserved": 1720073671192,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
@ -469,10 +471,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -480,7 +483,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327546135,
"lastObserved": 1720073671209,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
@ -1217,10 +1220,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -1228,7 +1232,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327546161,
"lastObserved": 1720073671300,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
@ -1414,10 +1418,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -1425,7 +1430,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327546168,
"lastObserved": 1720073671321,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}

25
metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json
View File

@ -87,10 +87,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -98,7 +99,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327442987,
"lastObserved": 1720073670616,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
@ -303,10 +304,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -314,7 +316,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327443016,
"lastObserved": 1720073670661,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
@ -432,10 +434,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -443,7 +446,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327443023,
"lastObserved": 1720073670669,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
@ -1113,10 +1116,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -1124,7 +1128,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327443058,
"lastObserved": 1720073670731,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}
@ -1289,10 +1293,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:Shubham jagtap",
"owner": "urn:li:corpuser:john doe",
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -1300,7 +1305,7 @@
}
},
"systemMetadata": {
"lastObserved": 1707327443073,
"lastObserved": 1720073670744,
"runId": "qlik-sense-test",
"lastRunId": "no-run-id-provided"
}

4
metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py
View File

@ -204,8 +204,8 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A",
"status": "active",
"subject": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f",
"name": "Shubham jagtap",
"email": "Shubham.Jagtap@gslab.com",
"name": "john doe",
"email": "john.doe@example.com",
"roles": [
"TenantAdmin",
"AnalyticsAdmin",

8
metadata-ingestion/tests/integration/sigma/test_sigma.py
View File

@ -264,7 +264,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"json": {
"elementId": "kH0MeihtGs",
"name": "ADOPTIONS",
"sql": 'select PK "Pk", PET_FK "Pet Fk", HUMAN_FK "Human Fk", STATUS "Status", CAST_TIMESTAMP_TO_DATETIME_7 "Created At", CAST_TIMESTAMP_TO_DATETIME_8 "Updated At" from (select PK, PET_FK, HUMAN_FK, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_8 from (select * from LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS ADOPTIONS limit 1000) Q1) Q2 limit 1000\n\n-- Sigma Σ {"request-id":"3d4bf15e-6a17-4967-ad2a-213341233a04","email":"Shubham.Jagtap@gslab.com"}',
"sql": 'select PK "Pk", PET_FK "Pet Fk", HUMAN_FK "Human Fk", STATUS "Status", CAST_TIMESTAMP_TO_DATETIME_7 "Created At", CAST_TIMESTAMP_TO_DATETIME_8 "Updated At" from (select PK, PET_FK, HUMAN_FK, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_8 from (select * from LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS ADOPTIONS limit 1000) Q1) Q2 limit 1000\n\n-- Sigma Σ {"request-id":"3d4bf15e-6a17-4967-ad2a-213341233a04","email":"john.doe@example.com"}',
},
},
"https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/lineage/elements/Ml9C5ezT5W": {
@ -299,7 +299,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"json": {
"elementId": "Ml9C5ezT5W",
"name": "Count of Profile Id by Status",
"sql": 'with Q1 as (select PK, PROFILE_ID, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_6, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) select STATUS_10 "Status", COUNT_23 "Count of Profile Id", PK_8 "Pk", PROFILE_ID_9 "Profile Id", CAST_TIMESTAMP_TO_DATETIME_8 "Created At", CAST_TIMESTAMP_TO_DATETIME_9 "Updated At" from (select Q3.PK_8 PK_8, Q3.PROFILE_ID_9 PROFILE_ID_9, Q3.STATUS_10 STATUS_10, Q3.CAST_TIMESTAMP_TO_DATETIME_8 CAST_TIMESTAMP_TO_DATETIME_8, Q3.CAST_TIMESTAMP_TO_DATETIME_9 CAST_TIMESTAMP_TO_DATETIME_9, Q6.COUNT_23 COUNT_23, Q6.STATUS_11 STATUS_11 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CAST_TIMESTAMP_TO_DATETIME_6 CAST_TIMESTAMP_TO_DATETIME_8, CAST_TIMESTAMP_TO_DATETIME_7 CAST_TIMESTAMP_TO_DATETIME_9 from Q1 Q2 order by STATUS_10 asc limit 1000) Q3 left join (select count(PROFILE_ID_9) COUNT_23, STATUS_10 STATUS_11 from (select PROFILE_ID PROFILE_ID_9, STATUS STATUS_10 from Q1 Q4) Q5 group by STATUS_10) Q6 on equal_null(Q3.STATUS_10, Q6.STATUS_11)) Q8 order by STATUS_10 asc limit 1000\n\n-- Sigma Σ {"request-id":"988dd6b5-0678-4421-ae14-21594c0ee97a","email":"Shubham.Jagtap@gslab.com"}',
"sql": 'with Q1 as (select PK, PROFILE_ID, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_6, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) select STATUS_10 "Status", COUNT_23 "Count of Profile Id", PK_8 "Pk", PROFILE_ID_9 "Profile Id", CAST_TIMESTAMP_TO_DATETIME_8 "Created At", CAST_TIMESTAMP_TO_DATETIME_9 "Updated At" from (select Q3.PK_8 PK_8, Q3.PROFILE_ID_9 PROFILE_ID_9, Q3.STATUS_10 STATUS_10, Q3.CAST_TIMESTAMP_TO_DATETIME_8 CAST_TIMESTAMP_TO_DATETIME_8, Q3.CAST_TIMESTAMP_TO_DATETIME_9 CAST_TIMESTAMP_TO_DATETIME_9, Q6.COUNT_23 COUNT_23, Q6.STATUS_11 STATUS_11 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CAST_TIMESTAMP_TO_DATETIME_6 CAST_TIMESTAMP_TO_DATETIME_8, CAST_TIMESTAMP_TO_DATETIME_7 CAST_TIMESTAMP_TO_DATETIME_9 from Q1 Q2 order by STATUS_10 asc limit 1000) Q3 left join (select count(PROFILE_ID_9) COUNT_23, STATUS_10 STATUS_11 from (select PROFILE_ID PROFILE_ID_9, STATUS STATUS_10 from Q1 Q4) Q5 group by STATUS_10) Q6 on equal_null(Q3.STATUS_10, Q6.STATUS_11)) Q8 order by STATUS_10 asc limit 1000\n\n-- Sigma Σ {"request-id":"988dd6b5-0678-4421-ae14-21594c0ee97a","email":"john.doe@example.com"}',
},
},
"https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/pages/DFSieiAcgo/elements": {
@ -375,7 +375,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"json": {
"elementId": "tQJu5N1l81",
"name": "PETS ADOPTIONS JOIN",
"sql": 'select PK_8 "Pk", PROFILE_ID_9 "Profile Id", STATUS_10 "Status", CAST_TIMESTAMP_TO_DATETIME_11 "Created At", CAST_TIMESTAMP_TO_DATETIME_12 "Updated At", PK_13 "Pk (ADOPTIONS)", PET_FK_14 "Pet Fk", HUMAN_FK_15 "Human Fk", STATUS_16 "Status (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_19 "Created At (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_20 "Updated At (ADOPTIONS)" from (select PK_8, PROFILE_ID_9, STATUS_10, CAST_TIMESTAMP_TO_DATETIME_11, CAST_TIMESTAMP_TO_DATETIME_12, PK_13, PET_FK_14, HUMAN_FK_15, STATUS_16, CREATED_AT_17::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_19, UPDATED_AT_18::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_20 from (select Q1.PK_8 PK_8, Q1.PROFILE_ID_9 PROFILE_ID_9, Q1.STATUS_10 STATUS_10, Q1.CAST_TIMESTAMP_TO_DATETIME_11 CAST_TIMESTAMP_TO_DATETIME_11, Q1.CAST_TIMESTAMP_TO_DATETIME_12 CAST_TIMESTAMP_TO_DATETIME_12, Q2.PK PK_13, Q2.PET_FK PET_FK_14, Q2.HUMAN_FK HUMAN_FK_15, Q2.STATUS STATUS_16, Q2.CREATED_AT CREATED_AT_17, Q2.UPDATED_AT UPDATED_AT_18 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_11, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_12 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) Q1 inner join LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS Q2 on (Q1.PK_8 = Q2.PET_FK) limit 1000) Q4) Q5 limit 1000\n\n-- Sigma Σ {"request-id":"f5a997ef-b80c-47f1-b32e-9cd0f50cd491","email":"Shubham.Jagtap@gslab.com"}',
"sql": 'select PK_8 "Pk", PROFILE_ID_9 "Profile Id", STATUS_10 "Status", CAST_TIMESTAMP_TO_DATETIME_11 "Created At", CAST_TIMESTAMP_TO_DATETIME_12 "Updated At", PK_13 "Pk (ADOPTIONS)", PET_FK_14 "Pet Fk", HUMAN_FK_15 "Human Fk", STATUS_16 "Status (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_19 "Created At (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_20 "Updated At (ADOPTIONS)" from (select PK_8, PROFILE_ID_9, STATUS_10, CAST_TIMESTAMP_TO_DATETIME_11, CAST_TIMESTAMP_TO_DATETIME_12, PK_13, PET_FK_14, HUMAN_FK_15, STATUS_16, CREATED_AT_17::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_19, UPDATED_AT_18::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_20 from (select Q1.PK_8 PK_8, Q1.PROFILE_ID_9 PROFILE_ID_9, Q1.STATUS_10 STATUS_10, Q1.CAST_TIMESTAMP_TO_DATETIME_11 CAST_TIMESTAMP_TO_DATETIME_11, Q1.CAST_TIMESTAMP_TO_DATETIME_12 CAST_TIMESTAMP_TO_DATETIME_12, Q2.PK PK_13, Q2.PET_FK PET_FK_14, Q2.HUMAN_FK HUMAN_FK_15, Q2.STATUS STATUS_16, Q2.CREATED_AT CREATED_AT_17, Q2.UPDATED_AT UPDATED_AT_18 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_11, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_12 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) Q1 inner join LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS Q2 on (Q1.PK_8 = Q2.PET_FK) limit 1000) Q4) Q5 limit 1000\n\n-- Sigma Σ {"request-id":"f5a997ef-b80c-47f1-b32e-9cd0f50cd491","email":"john.doe@example.com"}',
},
},
"https://aws-api.sigmacomputing.com/v2/members": {
@ -388,7 +388,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"memberType": "admin",
"firstName": "Shubham",
"lastName": "Jagtap",
"email": "Shubham.Jagtap@gslab.com",
"email": "john.doe@example.com",
"profileImgUrl": None,
"createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1",
"updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1",

64
metadata-ingestion/tests/unit/test_transform_dataset.py
View File

@ -1941,7 +1941,7 @@ def test_simple_add_dataset_domain_aspect_name(mock_datahub_graph):
def test_simple_add_dataset_domain(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
pipeline_context: PipelineContext = PipelineContext(
run_id="test_simple_add_dataset_domain"
@ -1950,7 +1950,7 @@ def test_simple_add_dataset_domain(mock_datahub_graph):
output = run_dataset_transformer_pipeline(
transformer_type=SimpleAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={"domains": [acryl_domain]},
pipeline_context=pipeline_context,
)
@ -1963,13 +1963,13 @@ def test_simple_add_dataset_domain(mock_datahub_graph):
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 2
assert gslab_domain in transformed_aspect.domains
assert datahub_domain in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
def test_simple_add_dataset_domain_mce_support(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
pipeline_context: PipelineContext = PipelineContext(
run_id="test_simple_add_dataset_domain"
@ -1979,7 +1979,7 @@ def test_simple_add_dataset_domain_mce_support(mock_datahub_graph):
output = run_dataset_transformer_pipeline(
transformer_type=SimpleAddDatasetDomain,
aspect=None,
config={"domains": [gslab_domain, acryl_domain]},
config={"domains": [datahub_domain, acryl_domain]},
pipeline_context=pipeline_context,
use_mce=True,
)
@ -1994,13 +1994,13 @@ def test_simple_add_dataset_domain_mce_support(mock_datahub_graph):
assert isinstance(output[1].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[1].record.aspect)
assert len(transformed_aspect.domains) == 2
assert gslab_domain in transformed_aspect.domains
assert datahub_domain in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
pipeline_context: PipelineContext = PipelineContext(
run_id="test_simple_add_dataset_domain"
@ -2009,7 +2009,7 @@ def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph):
output = run_dataset_transformer_pipeline(
transformer_type=SimpleAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={"replace_existing": True, "domains": [acryl_domain]},
pipeline_context=pipeline_context,
)
@ -2022,13 +2022,13 @@ def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph):
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 1
assert gslab_domain not in transformed_aspect.domains
assert datahub_domain not in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
server_domain = builder.make_domain_urn("test.io")
pipeline_context = PipelineContext(run_id="transformer_pipe_line")
@ -2042,7 +2042,7 @@ def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph):
output = run_dataset_transformer_pipeline(
transformer_type=SimpleAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={
"semantics": TransformerSemantics.OVERWRITE,
"domains": [acryl_domain],
@ -2058,7 +2058,7 @@ def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph):
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 2
assert gslab_domain in transformed_aspect.domains
assert datahub_domain in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
assert server_domain not in transformed_aspect.domains
@ -2067,7 +2067,7 @@ def test_simple_add_dataset_domain_semantics_patch(
pytestconfig, tmp_path, mock_time, mock_datahub_graph
):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
server_domain = builder.make_domain_urn("test.io")
pipeline_context = PipelineContext(run_id="transformer_pipe_line")
@ -2081,7 +2081,7 @@ def test_simple_add_dataset_domain_semantics_patch(
output = run_dataset_transformer_pipeline(
transformer_type=SimpleAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={
"replace_existing": False,
"semantics": TransformerSemantics.PATCH,
@ -2098,7 +2098,7 @@ def test_simple_add_dataset_domain_semantics_patch(
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 3
assert gslab_domain in transformed_aspect.domains
assert datahub_domain in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
assert server_domain in transformed_aspect.domains
@ -2117,7 +2117,7 @@ def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph):
def test_pattern_add_dataset_domain_match(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*"
pipeline_context: PipelineContext = PipelineContext(
@ -2127,7 +2127,7 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph):
output = run_dataset_transformer_pipeline(
transformer_type=PatternAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={
"domain_pattern": {"rules": {pattern: [acryl_domain]}},
},
@ -2142,13 +2142,13 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph):
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 2
assert gslab_domain in transformed_aspect.domains
assert datahub_domain in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
def test_pattern_add_dataset_domain_no_match(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*"
pipeline_context: PipelineContext = PipelineContext(
@ -2158,7 +2158,7 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph):
output = run_dataset_transformer_pipeline(
transformer_type=PatternAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={
"domain_pattern": {"rules": {pattern: [acryl_domain]}},
},
@ -2173,13 +2173,13 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph):
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 1
assert gslab_domain in transformed_aspect.domains
assert datahub_domain in transformed_aspect.domains
assert acryl_domain not in transformed_aspect.domains
def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*"
pipeline_context: PipelineContext = PipelineContext(
@ -2189,7 +2189,7 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph):
output = run_dataset_transformer_pipeline(
transformer_type=PatternAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={
"replace_existing": True,
"domain_pattern": {"rules": {pattern: [acryl_domain]}},
@ -2205,13 +2205,13 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph):
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 1
assert gslab_domain not in transformed_aspect.domains
assert datahub_domain not in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*"
pipeline_context: PipelineContext = PipelineContext(
@ -2221,7 +2221,7 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph
output = run_dataset_transformer_pipeline(
transformer_type=PatternAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={
"replace_existing": True,
"domain_pattern": {"rules": {pattern: [acryl_domain]}},
@ -2241,7 +2241,7 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph
def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
server_domain = builder.make_domain_urn("test.io")
pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*"
@ -2256,7 +2256,7 @@ def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph):
output = run_dataset_transformer_pipeline(
transformer_type=PatternAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={
"semantics": TransformerSemantics.OVERWRITE,
"domain_pattern": {"rules": {pattern: [acryl_domain]}},
@ -2272,7 +2272,7 @@ def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph):
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 2
assert gslab_domain in transformed_aspect.domains
assert datahub_domain in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
assert server_domain not in transformed_aspect.domains
@ -2281,7 +2281,7 @@ def test_pattern_add_dataset_domain_semantics_patch(
pytestconfig, tmp_path, mock_time, mock_datahub_graph
):
acryl_domain = builder.make_domain_urn("acryl.io")
gslab_domain = builder.make_domain_urn("gslab.io")
datahub_domain = builder.make_domain_urn("datahubproject.io")
server_domain = builder.make_domain_urn("test.io")
pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*"
@ -2296,7 +2296,7 @@ def test_pattern_add_dataset_domain_semantics_patch(
output = run_dataset_transformer_pipeline(
transformer_type=PatternAddDatasetDomain,
aspect=models.DomainsClass(domains=[gslab_domain]),
aspect=models.DomainsClass(domains=[datahub_domain]),
config={
"replace_existing": False,
"semantics": TransformerSemantics.PATCH,
@ -2313,7 +2313,7 @@ def test_pattern_add_dataset_domain_semantics_patch(
assert isinstance(output[0].record.aspect, models.DomainsClass)
transformed_aspect = cast(models.DomainsClass, output[0].record.aspect)
assert len(transformed_aspect.domains) == 3
assert gslab_domain in transformed_aspect.domains
assert datahub_domain in transformed_aspect.domains
assert acryl_domain in transformed_aspect.domains
assert server_domain in transformed_aspect.domains