diff --git a/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql index b0b73d947fc..a9a9b3b3b91 100644 --- a/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql @@ -1,3 +1,17 @@ UPDATE test_case SET json = json_set(json, '$.createdBy', json->>'$.updatedBy') -WHERE json->>'$.createdBy' IS NULL; \ No newline at end of file +WHERE json->>'$.createdBy' IS NULL; + +-- Made httpPath a required field for Databricks, updating records +-- where httpPath is NULL or missing to an empty string. +UPDATE + dbservice_entity +SET + json = JSON_SET(json, '$.connection.config.httpPath', '') +WHERE + serviceType = 'Databricks' + AND ( + JSON_CONTAINS_PATH(json, 'one', '$.connection.config.httpPath') = 0 + OR JSON_UNQUOTE(json ->> '$.connection.config.httpPath') IS NULL + OR json ->> '$.connection.config.httpPath' = 'null' + ); \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.7.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.7.0/postgres/postDataMigrationSQLScript.sql index 459728b5b11..bf284243bec 100644 --- a/bootstrap/sql/migrations/native/1.7.0/postgres/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.7.0/postgres/postDataMigrationSQLScript.sql @@ -1,3 +1,23 @@ UPDATE test_case SET json = json || jsonb_build_object('createdBy', json->>'updatedBy') -WHERE json->>'createdBy' IS NULL; \ No newline at end of file +WHERE json->>'createdBy' IS NULL; + +-- Made httpPath a required field for Databricks, updating records +-- where httpPath is NULL or missing to an empty string. +UPDATE + dbservice_entity +SET + json = jsonb_set( + json, + '{connection,config,httpPath}', + '""' :: jsonb, + true + ) +WHERE + serviceType = 'Databricks' + AND ( + NOT jsonb_path_exists(json, '$.connection.config.httpPath') + OR (json -> 'connection' -> 'config' ->> 'httpPath') IS NULL + OR (json -> 'connection' -> 'config' ->> 'httpPath') = 'null' + ); + \ No newline at end of file diff --git a/ingestion/tests/unit/test_databricks_lineage.py b/ingestion/tests/unit/test_databricks_lineage.py index 1ebd777c442..99b38063990 100644 --- a/ingestion/tests/unit/test_databricks_lineage.py +++ b/ingestion/tests/unit/test_databricks_lineage.py @@ -87,6 +87,7 @@ mock_databricks_config = { "config": { "token": "random_token", "hostPort": "localhost:443", + "httpPath": "sql/1.0/endpoints/path", "connectionArguments": { "http_path": "sql/1.0/endpoints/path", }, diff --git a/ingestion/tests/unit/test_source_connection.py b/ingestion/tests/unit/test_source_connection.py index f244ac025bd..ff6b6a9414f 100644 --- a/ingestion/tests/unit/test_source_connection.py +++ b/ingestion/tests/unit/test_source_connection.py @@ -129,6 +129,7 @@ class SourceConnectionTest(TestCase): scheme=DatabricksScheme.databricks_connector, hostPort="1.1.1.1:443", token="KlivDTACWXKmZVfN1qIM", + httpPath="/sql/1.0/warehouses/abcdedfg", ) assert expected_result == get_connection_url(databricks_conn_obj) @@ -144,6 +145,7 @@ class SourceConnectionTest(TestCase): scheme=DatabricksScheme.databricks_connector, hostPort="1.1.1.1:443", token="KlivDTACWXKmZVfN1qIM", + httpPath="/sql/1.0/warehouses/abcdedfg", ) assert expected_result == get_connection_url(databricks_conn_obj) diff --git a/ingestion/tests/unit/test_source_parsing.py b/ingestion/tests/unit/test_source_parsing.py index 5eec5d8ed23..25b83d5e5d1 100644 --- a/ingestion/tests/unit/test_source_parsing.py +++ b/ingestion/tests/unit/test_source_parsing.py @@ -237,6 +237,7 @@ def test_databricks(): "config": { "token": "", "hostPort": "localhost:443", + "httpPath": "", "connectionArguments": { "http_path": "" }, diff --git a/ingestion/tests/unit/topology/database/test_databricks.py b/ingestion/tests/unit/topology/database/test_databricks.py index ecb49ea849a..916984ab319 100644 --- a/ingestion/tests/unit/topology/database/test_databricks.py +++ b/ingestion/tests/unit/topology/database/test_databricks.py @@ -48,6 +48,7 @@ mock_databricks_config = { "databaseSchema": "default", "token": "123sawdtesttoken", "hostPort": "localhost:443", + "httpPath": "/sql/1.0/warehouses/abcdedfg", "connectionArguments": {"http_path": "/sql/1.0/warehouses/abcdedfg"}, } }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json index a8b27f9f0a3..acb4fa35104 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json @@ -9,13 +9,17 @@ "databricksType": { "description": "Service type.", "type": "string", - "enum": ["Databricks"], + "enum": [ + "Databricks" + ], "default": "Databricks" }, "databricksScheme": { "description": "SQLAlchemy driver scheme options.", "type": "string", - "enum": ["databricks+connector"], + "enum": [ + "databricks+connector" + ], "default": "databricks+connector" } }, @@ -84,7 +88,9 @@ "$ref": "../../../../type/filterPattern.json#/definitions/filterPattern", "default": { "includes": [], - "excludes": ["^information_schema$"] + "excludes": [ + "^information_schema$" + ] } }, "tableFilterPattern": { @@ -98,7 +104,9 @@ "$ref": "../../../../type/filterPattern.json#/definitions/filterPattern", "default": { "includes": [], - "excludes": ["^system$"] + "excludes": [ + "^system$" + ] } }, "supportsUsageExtraction": { @@ -132,5 +140,9 @@ } }, "additionalProperties": false, - "required": ["hostPort", "token"] -} + "required": [ + "hostPort", + "token", + "httpPath" + ] +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/databricksConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/databricksConnection.ts index 24dcabcef0b..a4b962af4e2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/databricksConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/databricksConnection.ts @@ -45,7 +45,7 @@ export interface DatabricksConnection { /** * Databricks compute resources URL. */ - httpPath?: string; + httpPath: string; /** * Table name to fetch the query history. */