yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-24 14:08:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: making token as optional field (#9314)

Browse Source
This commit is contained in:
NiharDoshi99 2022-12-16 19:47:47 +05:30 committed by GitHub
parent 9598694b0d
commit 3190ac2516
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 37 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bootstrap/sql/com.mysql.cj.jdbc.Driver/v007__create_db_connection_info.sql
View File

@ -47,3 +47,11 @@ SET json = JSON_INSERT(
-- Remove DBT source config
UPDATE ingestion_pipeline_entity
SET json = JSON_REMOVE(json ,'$.sourceConfig.config.dbtConfigSource');
UPDATE pipeline_service_entity
SET json = JSON_INSERT(JSON_INSERT(JSON_REMOVE(json, '$.connection.config.configSource'),'$.connection.config.host', JSON_EXTRACT(json,'$.connection.config.configSource.host')),'$.connection.config.token',JSON_EXTRACT(json, '$.connection.config.configSource.token'))
WHERE serviceType = 'Dagster' AND json -> '$.connection.config.configSource.host' IS NOT NULL;
UPDATE pipeline_service_entity
SET json = JSON_INSERT(JSON_INSERT(JSON_REMOVE(json, '$.connection.config.configSource'),'$.connection.config.host', JSON_EXTRACT(json,'$.connection.config.configSource.hostPort')), '$.connection.config.token','')
WHERE serviceType = 'Dagster' AND json -> '$.connection.config.configSource.hostPort' IS NOT NULL;

8
bootstrap/sql/org.postgresql.Driver/v007__create_db_connection_info.sql
View File

@ -44,3 +44,11 @@ where serviceType in ('Db2');
UPDATE ingestion_pipeline_entity
SET json = json::jsonb #- '{sourceConfig,config,dbtConfigSource}';
UPDATE pipeline_service_entity
SET json = jsonb_set(jsonb_set(json::jsonb #- '{connection,config,configSource}', '{connection,config,token}', json#> '{connection,config,configSource,token}', true) ,'{connection,config,host}', json #> '{connection,config,configSource,host}' , true)
WHERE serviceType = 'Dagster' and json #>'{connection,config,configSource,host}' is not null;
update pipeline_service_entity
set json = jsonb_set(json::jsonb #- '{connection,config,configSource}', '{connection,config,host}', json#> '{connection,config,configSource,hostPort}', true)
where servicetype = 'Dagster' and json #>'{connection,config,configSource,hostPort}' is not null;

3
ingestion/src/metadata/clients/connection_clients.py
View File

@ -142,9 +142,8 @@ class FivetranClient:
@dataclass
class DagsterClient:
def __init__(self, client, config) -> None:
def __init__(self, client) -> None:
self.client = client
self.config = config
@dataclass

76
ingestion/src/metadata/utils/connections.py
View File

@ -149,9 +149,7 @@ from metadata.generated.schema.entity.services.connections.pipeline.backendConne
BackendConnection,
)
from metadata.generated.schema.entity.services.connections.pipeline.dagsterConnection import (
CloudDagster,
DagsterConnection,
LocalDagtser,
)
from metadata.generated.schema.entity.services.connections.pipeline.domoPipelineConnection import (
DomoPipelineConnection,
@ -1187,89 +1185,29 @@ def _(connection: DagsterClient) -> None:
from metadata.utils.graphql_queries import TEST_QUERY_GRAPHQL
try:
config = connection.config.configSource
if isinstance(config, LocalDagtser):
from urllib.parse import urlparse
from dagster_graphql import DagsterGraphQLClient
hostPort = config.hostPort # pylint: disable=invalid-name
hostPort = urlparse(hostPort) # pylint: disable=invalid-name
local_dagster = DagsterGraphQLClient(
hostname=hostPort.hostname, port_number=hostPort.port
)
local_dagster._execute( # pylint: disable=protected-access
TEST_QUERY_GRAPHQL
)
if isinstance(config, CloudDagster):
from dagster_graphql import DagsterGraphQLClient
from gql.transport.requests import RequestsHTTPTransport
url = config.host
cloud_dagster = DagsterGraphQLClient(
url,
transport=RequestsHTTPTransport(
url=url + "/graphql",
headers={
"Dagster-Cloud-Api-Token": config.token.get_secret_value()
},
),
)
cloud_dagster._execute( # pylint: disable=protected-access
TEST_QUERY_GRAPHQL
)
connection._execute(TEST_QUERY_GRAPHQL) # pylint: disable=protected-access
except Exception as exc:
msg = f"Unknown error connecting with {connection}: {exc}."
raise SourceConnectionException(msg) from exc
@singledispatch
def get_dagster_client(config):
"""
Method to retrieve dagster client from the config
"""
if config:
msg = f"Config not implemented for type {type(config)}: {config}"
raise NotImplementedError(msg)
@get_connection.register
def _(connection: DagsterConnection) -> DagsterClient:
dagster_connection = get_dagster_client(connection.configSource)
return DagsterClient(client=dagster_connection, config=connection)
@get_dagster_client.register
def _(config: LocalDagtser):
from urllib.parse import urlparse
from dagster_graphql import DagsterGraphQLClient
host_port = config.hostPort
host_port = urlparse(host_port)
local_dagster = DagsterGraphQLClient(
hostname=host_port.hostname, port_number=host_port.port
)
return local_dagster
@get_dagster_client.register
def _(config: CloudDagster):
from dagster_graphql import DagsterGraphQLClient
from gql.transport.requests import RequestsHTTPTransport
url = config.host
cloud_dagster = DagsterGraphQLClient(
url = connection.host
dagster_connection = DagsterGraphQLClient(
url,
transport=RequestsHTTPTransport(
url=f"{url}/graphql",
headers={"Dagster-Cloud-Api-Token": config.token.get_secret_value()},
headers={"Dagster-Cloud-Api-Token": connection.token.get_secret_value()}
if connection.token
else None,
),
)
return cloud_dagster
return DagsterClient(dagster_connection)
@get_connection.register

7
ingestion/tests/unit/topology/pipeline/test_dagster.py
View File

@ -48,12 +48,7 @@ mock_dagster_config = {
"type": "dagster",
"serviceName": "dagster_source",
"serviceConnection": {
"config": {
"type": "Dagster",
"configSource": {
"hostPort": "http://lolhost:3000",
},
}
"config": {"type": "Dagster", "host": "http://lolhost:3000"}
},
"sourceConfig": {"config": {"type": "PipelineMetadata"}},
},

58
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/dagsterConnection.json
View File

@ -11,40 +11,6 @@
"type": "string",
"enum": ["Dagster"],
"default": "Dagster"
},
"LocalDagtser": {
"title": "Local Dagster Config Source",
"description": "Config to connect to local Dagster",
"type": "object",
"properties": {
"hostPort": {
"title": "Host and Port",
"description": "Pipeline Service Management/UI URI.",
"type": "string",
"format": "uri"
}
},
"required": ["hostPort"]
},
"CloudDagster": {
"title": "Cloud Dagster Config Source",
"description": "Config to connect to Cloud Dagster",
"type": "object",
"properties": {
"host": {
"title": "Host",
"description": "Pipeline Service Management/UI URI.",
"type": "string",
"format": "uri"
},
"token": {
"title": "Token",
"description": "To Connect to Dagster Cloud",
"type": "string",
"format": "password"
}
},
"required": ["host", "token"]
}
},
"properties": {
@ -54,17 +20,17 @@
"$ref": "#/definitions/DagsterType",
"default": "Dagster"
},
"configSource": {
"title": "Dagster Configuration Source",
"description": "Available sources to fetch files.",
"oneOf": [
{
"$ref": "#/definitions/LocalDagtser"
},
{
"$ref": "#/definitions/CloudDagster"
}
]
"host": {
"title": "Host",
"description": "URL to the Dagster instance",
"type": "string",
"format": "uri"
},
"token": {
"title": "Token",
"description": "To Connect to Dagster Cloud",
"type": "string",
"format": "password"
},
"supportsMetadataExtraction": {
"title": "Supports Metadata Extraction",
@ -72,5 +38,5 @@
}
},
"additionalProperties": false,
"required": ["configSource"]
"required": ["host"]
}