yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-06-27 04:22:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes 11068: mysql & postgres iam auth (#11937)

Browse Source
This commit is contained in:
Ayush Shah 2023-06-16 13:18:12 +05:30 committed by GitHub
parent 8ad8001da1
commit f80eaf3a26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
55 changed files with 657 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
bootstrap/sql/com.mysql.cj.jdbc.Driver/v013__create_db_connection_info.sql
View File

@ -109,3 +109,15 @@ SET pe.json = JSON_INSERT(
'$.tasks',
updated_data.updated_json
);
-- Modify migrations for service connection of postgres and mysql to move password under authType
UPDATE dbservice_entity
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.connection.config.password'),
'$.connection.config.authType',
JSON_OBJECT(),
'$.connection.config.authType.password',
JSON_EXTRACT(json, '$.connection.config.password'))
where serviceType in ('Postgres', 'Mysql');

12
bootstrap/sql/org.postgresql.Driver/v013__create_db_connection_info.sql
View File

@ -72,3 +72,15 @@ SET json = jsonb_set(
FROM jsonb_array_elements(json->'tasks') AS t
)
);
-- Modify migrations for service connection of postgres and mysql to move password under authType
UPDATE dbservice_entity
SET json = jsonb_set(
json #-'{connection,config,password}',
'{connection,config,authType}',
jsonb_build_object('password',json#>'{connection,config,password}')
)
WHERE serviceType IN ('Postgres', 'Mysql')
and json#>'{connection,config,password}' is not null;

4
ingestion/src/metadata/clients/aws_client.py
View File

@ -34,6 +34,7 @@ class AWSServices(Enum):
KINESIS = "kinesis"
QUICKSIGHT = "quicksight"
ATHENA = "athena"
RDS = "rds"
class AWSAssumeRoleException(Exception):
@ -162,6 +163,9 @@ class AWSClient:
)
return session.resource(service_name=service_name)
def get_rds_client(self):
return self.get_client(AWSServices.RDS.value)
def get_s3_client(self):
return self.get_client(AWSServices.S3.value)

43
ingestion/src/metadata/ingestion/connections/builders.py
View File

@ -22,12 +22,17 @@ from sqlalchemy.engine import Engine
from sqlalchemy.event import listen
from sqlalchemy.pool import QueuePool
from metadata.clients.aws_client import AWSClient
from metadata.generated.schema.entity.services.connections.connectionBasicType import (
ConnectionArguments,
ConnectionOptions,
)
from metadata.generated.schema.entity.services.connections.database.common.iamAuthConfig import (
IamAuthConfigurationSource,
)
from metadata.ingestion.connections.headers import inject_query_header_by_conn
from metadata.ingestion.connections.secrets import connection_with_options_secrets
from metadata.utils.constants import BUILDER_PASSWORD_ATTR
@connection_with_options_secrets
@ -117,6 +122,40 @@ def init_empty_connection_options() -> ConnectionOptions:
return ConnectionOptions(__root__={})
def _add_password(url: str, connection) -> str:
"""
A helper function that adds the password to the url if it exists.
Distinguishing between BasicAuth (Password) and IamAuth (AWSConfig)
and adding to url.
"""
password = getattr(connection, BUILDER_PASSWORD_ATTR, None)
if not password:
password = SecretStr("")
# Check if IamAuth exists - specific to Mysql and Postgres connection.
if hasattr(connection, "authType"):
password = getattr(
connection.authType, BUILDER_PASSWORD_ATTR, SecretStr("")
)
if isinstance(connection.authType, IamAuthConfigurationSource):
# if IAM based, fetch rds client and generate db auth token.
aws_client = AWSClient(
config=connection.authType.awsConfig
).get_rds_client()
host, port = connection.hostPort.split(":")
password = SecretStr(
aws_client.generate_db_auth_token(
DBHostname=host,
Port=port,
DBUsername=connection.username,
Region=connection.authType.awsConfig.awsRegion,
)
)
url += f":{quote_plus(password.get_secret_value())}"
return url
def get_connection_url_common(connection) -> str:
"""
Common method for building the source connection urls
@ -126,9 +165,7 @@ def get_connection_url_common(connection) -> str:
if connection.username:
url += f"{quote_plus(connection.username)}"
if not connection.password:
connection.password = SecretStr("")
url += f":{quote_plus(connection.password.get_secret_value())}"
url = _add_password(url, connection)
url += "@"
url += connection.hostPort

2
ingestion/src/metadata/utils/constants.py
View File

@ -18,7 +18,7 @@ TEN_MIN = 10 * 60
UTF_8 = "utf-8"
CHUNKSIZE = 200000
DEFAULT_DATABASE = "default"
BUILDER_PASSWORD_ATTR = "password"
ES_SOURCE_TO_ES_OBJ_ARGS = {
"caCerts": "ca_certs",

1
ingestion/tests/cli_e2e/database/mysql/mysql.yaml
View File

@ -5,6 +5,7 @@ source:
config:
type: Mysql
username: openmetadata_user
authType:
password: openmetadata_password
hostPort: localhost:3306
databaseSchema: openmetadata_db

5
ingestion/tests/integration/automations/test_connection_automation.py
View File

@ -28,6 +28,9 @@ from metadata.generated.schema.entity.automations.workflow import (
WorkflowType,
)
from metadata.generated.schema.entity.data.pipeline import StatusType
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
MySQLType,
@ -45,7 +48,7 @@ from metadata.ingestion.source.connections import get_connection, get_test_conne
SERVICE_CONNECTION = MysqlConnection(
username="openmetadata_user",
password="openmetadata_password",
authType=BasicAuth(password="openmetadata_password"),
hostPort="localhost:3306",
)

5
ingestion/tests/integration/connections/test_mysql_connection.py
View File

@ -16,6 +16,9 @@ from unittest import TestCase
from sqlalchemy.engine import Engine
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -29,7 +32,7 @@ class MySQLConnectionTest(TestCase):
connection = MysqlConnection(
username="openmetadata_user",
password="openmetadata_password",
authType=BasicAuth(password="openmetadata_password"),
hostPort="localhost:3306",
databaseSchema="openmetadata_db",
)

5
ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py
View File

@ -35,6 +35,9 @@ from metadata.generated.schema.api.services.createDatabaseService import (
)
from metadata.generated.schema.entity.data.pipeline import Pipeline, StatusType
from metadata.generated.schema.entity.data.table import Column, DataType, Table
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -100,7 +103,7 @@ class AirflowLineageTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
password="password",
authType=BasicAuth(password="password"),
hostPort="http://localhost:1234",
)
),

5
ingestion/tests/integration/ometa/test_ometa_database_api.py
View File

@ -21,6 +21,9 @@ from metadata.generated.schema.api.services.createDatabaseService import (
)
from metadata.generated.schema.api.teams.createUser import CreateUserRequest
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -69,7 +72,9 @@ class OMetaDatabaseTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

11
ingestion/tests/integration/ometa/test_ometa_database_service_api.py
View File

@ -19,6 +19,9 @@ from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -55,7 +58,11 @@ class OMetaDatabaseServiceTest(TestCase):
connection = DatabaseConnection(
config=MysqlConnection(
username="username", password="password", hostPort="http://localhost:1234"
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
)
@ -119,7 +126,9 @@ class OMetaDatabaseServiceTest(TestCase):
new_connection = DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:2000",
)
)

5
ingestion/tests/integration/ometa/test_ometa_es_api.py
View File

@ -24,6 +24,9 @@ from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.entity.data.table import Column, DataType, Table
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -65,7 +68,9 @@ class OMetaESTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

5
ingestion/tests/integration/ometa/test_ometa_lineage_api.py
View File

@ -28,6 +28,9 @@ from metadata.generated.schema.api.services.createPipelineService import (
CreatePipelineServiceRequest,
)
from metadata.generated.schema.entity.data.table import Column, DataType
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -84,7 +87,9 @@ class OMetaLineageTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

5
ingestion/tests/integration/ometa/test_ometa_model_api.py
View File

@ -37,6 +37,9 @@ from metadata.generated.schema.entity.data.mlmodel import (
MlModel,
)
from metadata.generated.schema.entity.data.table import Column, DataType, Table
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -270,7 +273,9 @@ class OMetaModelTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

5
ingestion/tests/integration/ometa/test_ometa_patch.py
View File

@ -29,6 +29,9 @@ from metadata.generated.schema.api.teams.createUser import CreateUserRequest
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.table import Column, DataType, Table
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -106,7 +109,9 @@ class OMetaTableTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

4
ingestion/tests/integration/ometa/test_ometa_service_api.py
View File

@ -79,7 +79,7 @@ class OMetaServiceTest(TestCase):
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType": {"password": "openmetadata_password"},
"hostPort": "random:3306",
}
},
@ -95,7 +95,7 @@ class OMetaServiceTest(TestCase):
assert service
assert service.serviceType == DatabaseServiceType.Mysql
assert (
service.connection.config.password.get_secret_value()
service.connection.config.authType.password.get_secret_value()
== "openmetadata_password"
)

5
ingestion/tests/integration/ometa/test_ometa_table_api.py
View File

@ -45,6 +45,9 @@ from metadata.generated.schema.entity.data.table import (
TableProfile,
TableProfilerConfig,
)
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -97,7 +100,9 @@ class OMetaTableTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

10
ingestion/tests/integration/ometa/test_ometa_workflow_api.py
View File

@ -26,6 +26,9 @@ from metadata.generated.schema.entity.automations.workflow import (
WorkflowStatus,
WorkflowType,
)
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
MySQLType,
@ -88,7 +91,9 @@ class OMetaWorkflowTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),
@ -108,7 +113,9 @@ class OMetaWorkflowTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),
@ -156,7 +163,8 @@ class OMetaWorkflowTest(TestCase):
# The ingestion-bot should see the password
self.assertEqual(
res.request.connection.config.password.get_secret_value(), "password"
res.request.connection.config.authType.password.get_secret_value(),
"password",
)
def test_get_id(self):

5
ingestion/tests/integration/orm_profiler/test_converter.py
View File

@ -25,6 +25,9 @@ from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.entity.data.table import Column, DataType
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -71,7 +74,9 @@ class ProfilerWorkflowTest(TestCase):
connection = DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
)

5
ingestion/tests/integration/orm_profiler/test_pii_processor.py
View File

@ -30,6 +30,9 @@ from metadata.generated.schema.entity.data.table import (
Table,
TableData,
)
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -286,7 +289,9 @@ class PiiProcessorTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

5
ingestion/tests/integration/utils/test_fqn.py
View File

@ -23,6 +23,9 @@ from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.entity.data.table import Column, DataType, Table
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -65,7 +68,9 @@ class FQNBuildTest(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

4
ingestion/tests/integration/workflow/mysql_test.json
View File

@ -7,7 +7,9 @@
"type": "Mysql",
"username": "openmetadata_user",
"database":"openmetadata_db",
"password": "openmetadata_password",
"authType":{
"password": "openmetadata_password"
},
"hostPort": "localhost:3306",
"connectionOptions": {},
"connectionArguments": {}

2
ingestion/tests/unit/source/test_source_parsing.py
View File

@ -525,7 +525,7 @@ def test_mysql():
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType": {"password": "openmetadata_password"},
"hostPort": "localhost:3306",
}
},

11
ingestion/tests/unit/test_connection_builders.py
View File

@ -13,6 +13,9 @@ Validate connection builder utilities
"""
from unittest import TestCase
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -29,19 +32,21 @@ class ConnectionBuilderTest(TestCase):
"""
connection = MysqlConnection(
username="username", password="password", hostPort="http://localhost:1234"
username="username",
authType=BasicAuth(password="password"),
hostPort="http://localhost:1234",
)
connection_with_args = MysqlConnection(
username="username",
password="password",
authType=BasicAuth(password="password"),
hostPort="http://localhost:1234",
connectionArguments={"hello": "world"},
)
connection_with_options = MysqlConnection(
username="username",
password="password",
authType=BasicAuth(password="password"),
hostPort="http://localhost:1234",
connectionOptions={"hello": "world"},
)

18
ingestion/tests/unit/test_source_connection.py
View File

@ -19,6 +19,9 @@ from metadata.generated.schema.entity.services.connections.database.clickhouseCo
ClickhouseConnection,
ClickhouseScheme,
)
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
DatabricksConnection,
DatabricksScheme,
@ -36,7 +39,6 @@ from metadata.generated.schema.entity.services.connections.database.hiveConnecti
HiveScheme,
)
from metadata.generated.schema.entity.services.connections.database.impalaConnection import (
AuthMechanism,
ImpalaConnection,
ImpalaScheme,
)
@ -654,8 +656,8 @@ class SourceConnectionTest(TestCase):
postgres_conn_obj = PostgresConnection(
username="openmetadata_user",
hostPort="localhost:5432",
scheme=PostgresScheme.postgresql_psycopg2,
database="default",
scheme=PostgresScheme.postgresql_psycopg2,
)
assert expected_url == get_connection_url_common(postgres_conn_obj)
@ -753,7 +755,9 @@ class SourceConnectionTest(TestCase):
expected_args = {}
mysql_conn_obj = MysqlConnection(
username="user",
authType=BasicAuth(
password=None,
),
hostPort="localhost:443",
connectionArguments=None,
scheme=MySQLScheme.mysql_pymysql,
@ -764,7 +768,9 @@ class SourceConnectionTest(TestCase):
expected_args = {"user": "user-to-be-impersonated"}
mysql_conn_obj = MysqlConnection(
username="user",
authType=BasicAuth(
password=None,
),
hostPort="localhost:443",
connectionArguments={"user": "user-to-be-impersonated"},
scheme=MySQLScheme.mysql_pymysql,
@ -824,9 +830,11 @@ class SourceConnectionTest(TestCase):
expected_args = {}
postgres_conn_obj = PostgresConnection(
username="user",
authType=BasicAuth(
password=None,
hostPort="localhost:443",
),
database="postgres",
hostPort="localhost:443",
connectionArguments=None,
scheme=PostgresScheme.postgresql_psycopg2,
)
@ -835,8 +843,10 @@ class SourceConnectionTest(TestCase):
# connection arguments with connectionArguments
expected_args = {"user": "user-to-be-impersonated"}
postgres_conn_obj = PostgresConnection(
username="user",
authType=BasicAuth(
password=None,
),
username="user",
hostPort="localhost:443",
database="tiny",
connectionArguments={"user": "user-to-be-impersonated"},

5
ingestion/tests/unit/test_source_url.py
View File

@ -14,6 +14,9 @@ OpenMetadata source URL building tests
"""
from unittest import TestCase
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
MssqlConnection,
MssqlScheme,
@ -34,7 +37,7 @@ class TestConfig(TestCase):
"""
connection = MysqlConnection(
username="username",
password="password",
authType=BasicAuth(password="password"),
hostPort="localhost:1234",
)
url = get_connection_url_common(connection)

6
ingestion/tests/unit/test_workflow_parse.py
View File

@ -320,7 +320,7 @@ class TestWorkflowParse(TestCase):
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType": {"password": "openmetadata_password"},
"hostPort": "localhost:3306",
}
},
@ -337,7 +337,7 @@ class TestWorkflowParse(TestCase):
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType": {"password": "openmetadata_password"},
}
},
"connectionType": "Database",
@ -355,7 +355,7 @@ class TestWorkflowParse(TestCase):
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType": {"password": "openmetadata_password"},
"hostPort": "localhost:3306",
"random": "value",
}

2
ingestion/tests/unit/topology/dashboard/test_looker.py
View File

@ -155,7 +155,7 @@ class LookerUnitTest(TestCase):
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType": {"password": "openmetadata_password"},
"hostPort": "localhost:3306",
"databaseSchema": "openmetadata_db",
}

15
ingestion/tests/unit/topology/dashboard/test_superset.py
View File

@ -23,6 +23,9 @@ from sqlalchemy.engine import Engine
from metadata.generated.schema.api.data.createChart import CreateChartRequest
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
from metadata.generated.schema.entity.data.chart import Chart, ChartType
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -110,7 +113,9 @@ MOCK_SUPERSET_DB_CONFIG = {
"connection": {
"type": "Postgres",
"username": "superset",
"authType": {
"password": "superset",
},
"hostPort": "localhost:5432",
"database": "superset",
},
@ -147,7 +152,9 @@ MOCK_DB_MYSQL_SERVICE_1 = DatabaseService(
name="test_mysql",
connection=DatabaseConnection(
config=MysqlConnection(
username="user", password="pass", hostPort="localhost:3306"
username="user",
authType=BasicAuth(password="pass"),
hostPort="localhost:3306",
)
),
serviceType=DatabaseServiceType.Mysql,
@ -160,7 +167,7 @@ MOCK_DB_MYSQL_SERVICE_2 = DatabaseService(
connection=DatabaseConnection(
config=MysqlConnection(
username="user",
password="pass",
authType=BasicAuth(password="pass"),
hostPort="localhost:3306",
databaseName="DUMMY_DB",
)
@ -175,7 +182,7 @@ MOCK_DB_POSTGRES_SERVICE = DatabaseService(
connection=DatabaseConnection(
config=PostgresConnection(
username="user",
password="pass",
authType=BasicAuth(password="pass"),
hostPort="localhost:5432",
database="postgres",
)
@ -282,7 +289,9 @@ class SupersetUnitTest(TestCase):
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"authType": {
"password": "openmetadata_password",
},
"hostPort": "localhost:3306",
"databaseSchema": "openmetadata_db",
}

4
ingestion/tests/unit/topology/database/test_postgres.py
View File

@ -50,7 +50,9 @@ mock_postgres_config = {
"config": {
"type": "Postgres",
"username": "username",
"authType": {
"password": "password",
},
"hostPort": "localhost:5432",
"database": "postgres",
}
@ -84,7 +86,9 @@ mock_postgres_usage_config = {
"config": {
"type": "Postgres",
"username": "username",
"authType": {
"password": "password",
},
"hostPort": "localhost:5432",
"database": "postgres",
}

2
ingestion/tests/unit/topology/storage/test_storage.py
View File

@ -126,7 +126,7 @@ class StorageUnitTest(TestCase):
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType": {"password": "openmetadata_password"},
"hostPort": "localhost:3306",
"databaseSchema": "openmetadata_db",
}

4
openmetadata-airflow-apis/examples/deploy_dag.json
View File

@ -14,7 +14,9 @@
"type": "MySQL",
"scheme": "mysql+pymysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType":{
"password": "openmetadata_password"
},
"hostPort": "localhost:3306",
"database": null,
"connectionOptions": null,

4
openmetadata-airflow-apis/examples/test_connection.json
View File

@ -3,7 +3,9 @@
"config": {
"type": "MySQL",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType":{
"password": "openmetadata_password"
},
"hostPort": "localhost:3306"
}
}

5
openmetadata-airflow-apis/tests/integration/operations/test_airflow_ops.py
View File

@ -24,6 +24,9 @@ from unittest import TestCase
from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
BasicAuth,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
@ -216,7 +219,9 @@ class TestAirflowOps(TestCase):
connection=DatabaseConnection(
config=MysqlConnection(
username="username",
authType=BasicAuth(
password="password",
),
hostPort="http://localhost:1234",
)
),

2
openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_workflow_creation.py
View File

@ -105,7 +105,7 @@ class OMetaServiceTest(TestCase):
"config": {
"type": "Mysql",
"username": "openmetadata_user",
"password": "openmetadata_password",
"authType": {"password": "openmetadata_password"},
"hostPort": "localhost:3306",
}
},

4
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java
View File

@ -26,6 +26,8 @@ import org.openmetadata.schema.services.connections.dashboard.SupersetConnection
import org.openmetadata.schema.services.connections.dashboard.TableauConnection;
import org.openmetadata.schema.services.connections.database.BigQueryConnection;
import org.openmetadata.schema.services.connections.database.DatalakeConnection;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.PostgresConnection;
import org.openmetadata.schema.services.connections.database.datalake.GCSConfig;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.schema.services.connections.pipeline.AirflowConnection;
@ -45,6 +47,8 @@ public final class ClassConverterFactory {
Map.entry(AirflowConnection.class, new AirflowConnectionClassConverter()),
Map.entry(BigQueryConnection.class, new BigQueryConnectionClassConverter()),
Map.entry(DatalakeConnection.class, new DatalakeConnectionClassConverter()),
Map.entry(MysqlConnection.class, new MysqlConnectionClassConverter()),
Map.entry(PostgresConnection.class, new PostgresConnectionClassConverter()),
Map.entry(DbtGCSConfig.class, new DbtGCSConfigClassConverter()),
Map.entry(DbtPipeline.class, new DbtPipelineClassConverter()),
Map.entry(GCSConfig.class, new GCPConfigClassConverter()),

39
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MysqlConnectionClassConverter.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.secrets.converter;
import java.util.List;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.common.IamAuthConfig;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.service.util.JsonUtils;
/** Converter class to get an `DatalakeConnection` object. */
public class MysqlConnectionClassConverter extends ClassConverter {
private static final List<Class<?>> CONFIG_SOURCE_CLASSES = List.of(basicAuth.class, IamAuthConfig.class);
public MysqlConnectionClassConverter() {
super(MysqlConnection.class);
}
@Override
public Object convert(Object object) {
MysqlConnection mysqlConnection = (MysqlConnection) JsonUtils.convertValue(object, this.clazz);
tryToConvert(mysqlConnection.getAuthType(), CONFIG_SOURCE_CLASSES).ifPresent(mysqlConnection::setAuthType);
return mysqlConnection;
}
}

39
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/PostgresConnectionClassConverter.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright 2021 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.service.secrets.converter;
import java.util.List;
import org.openmetadata.schema.services.connections.database.PostgresConnection;
import org.openmetadata.schema.services.connections.database.common.IamAuthConfig;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.service.util.JsonUtils;
/** Converter class to get an `DatalakeConnection` object. */
public class PostgresConnectionClassConverter extends ClassConverter {
private static final List<Class<?>> CONFIG_SOURCE_CLASSES = List.of(basicAuth.class, IamAuthConfig.class);
public PostgresConnectionClassConverter() {
super(PostgresConnection.class);
}
@Override
public Object convert(Object object) {
PostgresConnection postgresConnection = (PostgresConnection) JsonUtils.convertValue(object, this.clazz);
tryToConvert(postgresConnection.getAuthType(), CONFIG_SOURCE_CLASSES).ifPresent(postgresConnection::setAuthType);
return postgresConnection;
}
}

3
openmetadata-service/src/test/java/org/openmetadata/service/resources/automations/WorkflowResourceTest.java
View File

@ -16,6 +16,7 @@ import org.openmetadata.schema.entity.automations.Workflow;
import org.openmetadata.schema.entity.automations.WorkflowType;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.automations.WorkflowResource.WorkflowList;
@ -43,7 +44,7 @@ public class WorkflowResourceTest extends EntityResourceTest<Workflow, CreateWor
new MysqlConnection()
.withHostPort("mysql:3306")
.withUsername("openmetadata_user")
.withPassword("openmetadata_password"))));
.withAuthType(new basicAuth().withPassword("openmetadata_password")))));
}
@Override

9
openmetadata-service/src/test/java/org/openmetadata/service/resources/services/DatabaseServiceResourceTest.java
View File

@ -52,6 +52,7 @@ import org.openmetadata.schema.services.connections.database.ConnectionOptions;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.RedshiftConnection;
import org.openmetadata.schema.services.connections.database.SnowflakeConnection;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.schema.type.ChangeDescription;
import org.openmetadata.schema.type.Schedule;
import org.openmetadata.service.Entity;
@ -387,9 +388,13 @@ public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseServ
assertEquals(expectedMysqlConnection.getConnectionOptions(), actualMysqlConnection.getConnectionOptions());
assertEquals(expectedMysqlConnection.getConnectionArguments(), actualMysqlConnection.getConnectionArguments());
if (maskedPasswords) {
assertEquals(actualMysqlConnection.getPassword(), PasswordEntityMasker.PASSWORD_MASK);
assertEquals(
JsonUtils.convertValue(actualMysqlConnection.getAuthType(), basicAuth.class).getPassword(),
PasswordEntityMasker.PASSWORD_MASK);
} else {
assertEquals(expectedMysqlConnection.getPassword(), actualMysqlConnection.getPassword());
assertEquals(
JsonUtils.convertValue(expectedMysqlConnection.getAuthType(), basicAuth.class).getPassword(),
JsonUtils.convertValue(actualMysqlConnection.getAuthType(), basicAuth.class).getPassword());
}
}

23
openmetadata-service/src/test/java/org/openmetadata/service/secrets/ExternalSecretsManagerTest.java
View File

@ -41,6 +41,7 @@ import org.openmetadata.schema.security.secrets.Parameters;
import org.openmetadata.schema.security.secrets.SecretsManagerConfiguration;
import org.openmetadata.schema.security.secrets.SecretsManagerProvider;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.InvalidServiceConnectionException;
@ -67,20 +68,20 @@ public abstract class ExternalSecretsManagerTest {
@Test
void testEncryptDecryptDatabaseServiceConnectionConfig() {
String password = "openmetadata-test";
MysqlConnection expectedConnection = new MysqlConnection().withPassword(password);
Map<String, String> mysqlConnection = Map.of("password", password);
MysqlConnection expectedConnection = new MysqlConnection().withAuthType(new basicAuth().withPassword(password));
Map<String, Map<String, String>> mysqlConnection = Map.of("authType", Map.of("password", password));
// Ensure encrypted service connection config encrypts the password
MysqlConnection actualConnection =
(MysqlConnection)
secretsManager.encryptServiceConnectionConfig(mysqlConnection, Mysql.value(), "test", ServiceType.DATABASE);
assertNotEquals(password, actualConnection.getPassword());
assertNotEquals(password, JsonUtils.convertValue(actualConnection.getAuthType(), basicAuth.class).getPassword());
// Decrypt the encrypted password and validate
actualConnection =
(MysqlConnection)
secretsManager.decryptServiceConnectionConfig(mysqlConnection, Mysql.value(), ServiceType.DATABASE);
assertEquals(password, actualConnection.getPassword());
assertEquals(password, JsonUtils.convertValue(actualConnection.getAuthType(), basicAuth.class).getPassword());
assertEquals(expectedConnection, actualConnection);
}
@ -145,7 +146,8 @@ public abstract class ExternalSecretsManagerTest {
String secretKey = "secret:/openmetadata/serverconnection/securityconfig/secretkey";
OpenMetadataConnection connection =
new OpenMetadataConnection().withSecurityConfig(new GoogleSSOClientConfig().withSecretKey(secretKey));
DatabaseConnection dbConnection = new DatabaseConnection().withConfig(new MysqlConnection().withPassword(password));
DatabaseConnection dbConnection =
new DatabaseConnection().withConfig(new MysqlConnection().withAuthType(new basicAuth().withPassword(password)));
TestServiceConnectionRequest testRequest =
new TestServiceConnectionRequest()
.withConnection(dbConnection)
@ -169,7 +171,8 @@ public abstract class ExternalSecretsManagerTest {
@Test
void testExceptionConnection() {
Map<String, String> mysqlConnection = Map.of("password", "openmetadata-test", "username1", "openmetadata-test");
Map<String, Object> mysqlConnection =
Map.of("username1", "openmetadata-test", "authType", Map.of("password", "openmetadata-test"));
InvalidServiceConnectionException thrown =
Assertions.assertThrows(
InvalidServiceConnectionException.class,
@ -209,8 +212,12 @@ public abstract class ExternalSecretsManagerTest {
}
private String getPassword(Workflow workflow) {
return ((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) workflow.getRequest()).getConnection()).getConfig())
return JsonUtils.convertValue(
((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) workflow.getRequest()).getConnection())
.getConfig())
.getAuthType(),
basicAuth.class)
.getPassword();
}

14
openmetadata-service/src/test/java/org/openmetadata/service/secrets/NoopSecretsManagerTest.java
View File

@ -29,8 +29,10 @@ import org.mockito.junit.jupiter.MockitoExtension;
import org.openmetadata.schema.entity.services.ServiceType;
import org.openmetadata.schema.security.secrets.SecretsManagerProvider;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.schema.services.connections.mlmodel.SklearnConnection;
import org.openmetadata.service.fernet.Fernet;
import org.openmetadata.service.util.JsonUtils;
@ExtendWith(MockitoExtension.class)
public class NoopSecretsManagerTest {
@ -85,18 +87,22 @@ public class NoopSecretsManagerTest {
}
private void testEncryptServiceConnection() {
MysqlConnection connection = new MysqlConnection().withPassword(ENCRYPTED_VALUE);
MysqlConnection connection = new MysqlConnection().withAuthType(new basicAuth().withPassword(ENCRYPTED_VALUE));
Object actualConfig =
secretsManager.encryptServiceConnectionConfig(connection, Mysql.value(), "test", ServiceType.DATABASE);
assertEquals(ENCRYPTED_VALUE, ((MysqlConnection) actualConfig).getPassword());
assertEquals(
ENCRYPTED_VALUE,
JsonUtils.convertValue(((MysqlConnection) actualConfig).getAuthType(), basicAuth.class).getPassword());
assertNotSame(connection, actualConfig);
}
private void testDecryptServiceConnection() {
MysqlConnection mysqlConnection = new MysqlConnection().withPassword(DECRYPTED_VALUE);
MysqlConnection mysqlConnection = new MysqlConnection().withAuthType(new basicAuth().withPassword(DECRYPTED_VALUE));
Object actualConfig =
secretsManager.decryptServiceConnectionConfig(mysqlConnection, Mysql.value(), ServiceType.DATABASE);
assertEquals(DECRYPTED_VALUE, ((MysqlConnection) actualConfig).getPassword());
assertEquals(
DECRYPTED_VALUE,
JsonUtils.convertValue(((MysqlConnection) actualConfig).getAuthType(), basicAuth.class).getPassword());
assertNotSame(mysqlConnection, actualConfig);
}
}

6
openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java
View File

@ -17,6 +17,8 @@ import org.openmetadata.schema.services.connections.dashboard.SupersetConnection
import org.openmetadata.schema.services.connections.dashboard.TableauConnection;
import org.openmetadata.schema.services.connections.database.BigQueryConnection;
import org.openmetadata.schema.services.connections.database.DatalakeConnection;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.PostgresConnection;
import org.openmetadata.schema.services.connections.database.datalake.GCSConfig;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.schema.services.connections.pipeline.AirflowConnection;
@ -30,6 +32,8 @@ public class ClassConverterFactoryTest {
AirflowConnection.class,
BigQueryConnection.class,
DatalakeConnection.class,
MysqlConnection.class,
PostgresConnection.class,
DbtGCSConfig.class,
DbtPipeline.class,
GCSConfig.class,
@ -49,6 +53,6 @@ public class ClassConverterFactoryTest {
@Test
void testClassConvertedMapIsNotModified() {
assertEquals(ClassConverterFactory.getConverterMap().size(), 15);
assertEquals(ClassConverterFactory.getConverterMap().size(), 17);
}
}

4
openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/PasswordEntityMaskerTest.java
View File

@ -19,8 +19,8 @@ public class PasswordEntityMaskerTest extends TestEntityMasker {
@Test
void testExceptionConnection() {
Map<String, String> mysqlConnectionObject =
Map.of("password", "openmetadata-test", "username1", "openmetadata-test");
Map<String, Object> mysqlConnectionObject =
Map.of("authType", Map.of("password", "openmetadata-test"), "username1", "openmetadata-test");
EntityMaskException thrown =
Assertions.assertThrows(

36
openmetadata-service/src/test/java/org/openmetadata/service/secrets/masker/TestEntityMasker.java
View File

@ -26,9 +26,11 @@ import org.openmetadata.schema.services.connections.dashboard.SupersetConnection
import org.openmetadata.schema.services.connections.database.BigQueryConnection;
import org.openmetadata.schema.services.connections.database.DatalakeConnection;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.schema.services.connections.database.datalake.GCSConfig;
import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection;
import org.openmetadata.schema.services.connections.pipeline.AirflowConnection;
import org.openmetadata.service.util.JsonUtils;
abstract class TestEntityMasker {
@ -51,12 +53,18 @@ abstract class TestEntityMasker {
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(airflowConnection, "Airflow", ServiceType.PIPELINE);
assertNotNull(masked);
assertEquals(((MysqlConnection) masked.getConnection()).getPassword(), getMaskedPassword());
assertEquals(
(JsonUtils.convertValue(((MysqlConnection) masked.getConnection()).getAuthType(), basicAuth.class)
.getPassword()),
getMaskedPassword());
AirflowConnection unmasked =
(AirflowConnection)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(masked, airflowConnection, "Airflow", ServiceType.PIPELINE);
assertEquals(((MysqlConnection) unmasked.getConnection()).getPassword(), PASSWORD);
assertEquals(
JsonUtils.convertValue(((MysqlConnection) unmasked.getConnection()).getAuthType(), basicAuth.class)
.getPassword(),
PASSWORD);
}
@Test
@ -158,12 +166,17 @@ abstract class TestEntityMasker {
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(supersetConnection, "Superset", ServiceType.DASHBOARD);
assertNotNull(masked);
assertEquals(((MysqlConnection) masked.getConnection()).getPassword(), getMaskedPassword());
assertEquals(
JsonUtils.convertValue(((MysqlConnection) masked.getConnection()).getAuthType(), basicAuth.class).getPassword(),
getMaskedPassword());
SupersetConnection unmasked =
(SupersetConnection)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(masked, supersetConnection, "Superset", ServiceType.DASHBOARD);
assertEquals(((MysqlConnection) unmasked.getConnection()).getPassword(), PASSWORD);
assertEquals(
JsonUtils.convertValue(((MysqlConnection) unmasked.getConnection()).getAuthType(), basicAuth.class)
.getPassword(),
PASSWORD);
}
@Test
@ -179,8 +192,12 @@ abstract class TestEntityMasker {
Workflow masked = EntityMaskerFactory.createEntityMasker().maskWorkflow(workflow);
assertNotNull(masked);
assertEquals(
JsonUtils.convertValue(
((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) masked.getRequest()).getConnection()).getConfig())
((DatabaseConnection) ((TestServiceConnectionRequest) masked.getRequest()).getConnection())
.getConfig())
.getAuthType(),
basicAuth.class)
.getPassword(),
getMaskedPassword());
assertEquals(
@ -188,9 +205,12 @@ abstract class TestEntityMasker {
getMaskedPassword());
Workflow unmasked = EntityMaskerFactory.createEntityMasker().unmaskWorkflow(masked, workflow);
assertEquals(
JsonUtils.convertValue(
((MysqlConnection)
((DatabaseConnection) ((TestServiceConnectionRequest) unmasked.getRequest()).getConnection())
.getConfig())
.getAuthType(),
basicAuth.class)
.getPassword(),
PASSWORD);
assertEquals(
@ -206,12 +226,12 @@ abstract class TestEntityMasker {
EntityMaskerFactory.createEntityMasker()
.maskServiceConnectionConfig(mysqlConnection, "Mysql", ServiceType.DATABASE);
assertNotNull(masked);
assertEquals(masked.getPassword(), getMaskedPassword());
assertEquals(JsonUtils.convertValue(masked.getAuthType(), basicAuth.class).getPassword(), getMaskedPassword());
MysqlConnection unmasked =
(MysqlConnection)
EntityMaskerFactory.createEntityMasker()
.unmaskServiceConnectionConfig(masked, mysqlConnection, "Mysql", ServiceType.DATABASE);
assertEquals(unmasked.getPassword(), PASSWORD);
assertEquals(JsonUtils.convertValue(unmasked.getAuthType(), basicAuth.class).getPassword(), PASSWORD);
}
protected String getMaskedPassword() {
@ -223,7 +243,7 @@ abstract class TestEntityMasker {
}
private MysqlConnection buildMysqlConnection() {
return new MysqlConnection().withPassword(PASSWORD);
return new MysqlConnection().withAuthType(new basicAuth().withPassword(PASSWORD));
}
private GCSConfig buildGcsConfig() {

5
openmetadata-service/src/test/java/org/openmetadata/service/util/JsonUtilsTest.java
View File

@ -37,6 +37,7 @@ import org.openmetadata.schema.entity.services.DatabaseService;
import org.openmetadata.schema.entity.teams.Team;
import org.openmetadata.schema.services.connections.dashboard.TableauConnection;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
/** This test provides examples of how to use applyPatch */
@Slf4j
@ -133,7 +134,9 @@ class JsonUtilsTest {
DatabaseService databaseService =
new DatabaseService()
.withName("test")
.withConnection(new DatabaseConnection().withConfig(new MysqlConnection().withPassword("password")));
.withConnection(
new DatabaseConnection()
.withConfig(new MysqlConnection().withAuthType(new basicAuth().withPassword("password"))));
String actualJson = JsonUtils.pojoToMaskedJson(databaseService);
assertEquals(expectedJson, actualJson);
}

7
openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java
View File

@ -61,6 +61,7 @@ import org.openmetadata.schema.services.connections.database.BigQueryConnection;
import org.openmetadata.schema.services.connections.database.MysqlConnection;
import org.openmetadata.schema.services.connections.database.RedshiftConnection;
import org.openmetadata.schema.services.connections.database.SnowflakeConnection;
import org.openmetadata.schema.services.connections.database.common.basicAuth;
import org.openmetadata.schema.services.connections.messaging.KafkaConnection;
import org.openmetadata.schema.services.connections.messaging.RedpandaConnection;
import org.openmetadata.schema.services.connections.metadata.AmundsenConnection;
@ -142,7 +143,11 @@ public final class TestUtils {
static {
MYSQL_DATABASE_CONNECTION =
new DatabaseConnection()
.withConfig(new MysqlConnection().withHostPort("localhost:3306").withUsername("test").withPassword("test"));
.withConfig(
new MysqlConnection()
.withHostPort("localhost:3306")
.withUsername("test")
.withAuthType(new basicAuth().withPassword("test")));
REDSHIFT_DATABASE_CONNECTION =
new DatabaseConnection()
.withConfig(

17
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/common/basicAuth.json Normal file
View File

@ -0,0 +1,17 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/database/common/basicAuth.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Basic Auth",
"description": "Common Database Connection Config",
"javaType": "org.openmetadata.schema.services.connections.database.common.basicAuth",
"type": "object",
"properties": {
"password": {
"title": "Password",
"description": "Password to connect to source.",
"type": "string",
"format": "password"
}
},
"additionalProperties": false
}

15
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/common/iamAuthConfig.json Normal file
View File

@ -0,0 +1,15 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/database/common/iamAuthConfig.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "IAM Auth Configuration Source",
"description": "IAM Auth Database Connection Config",
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.database.common.IamAuthConfig",
"properties": {
"awsConfig": {
"title": "AWS Credentials Configuration",
"$ref": "../../../../../security/credentials/awsCredentials.json"
}
},
"additionalProperties": false
}

16
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json
View File

@ -37,11 +37,17 @@
"description": "Username to connect to MySQL. This user should have privileges to read all the metadata in Mysql.",
"type": "string"
},
"password": {
"title": "Password",
"description": "Password to connect to MySQL.",
"type": "string",
"format": "password"
"authType": {
"title": "Auth Configuration Type",
"description": "Choose Auth Config Type.",
"oneOf": [
{
"$ref": "./common/basicAuth.json"
},
{
"$ref": "./common/iamAuthConfig.json"
}
]
},
"hostPort": {
"title": "Host and Port",

41
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json
View File

@ -9,13 +9,17 @@
"postgresType": {
"description": "Service type.",
"type": "string",
"enum": ["Postgres"],
"enum": [
"Postgres"
],
"default": "Postgres"
},
"postgresScheme": {
"description": "SQLAlchemy driver scheme options.",
"type": "string",
"enum": ["postgresql+psycopg2"],
"enum": [
"postgresql+psycopg2"
],
"default": "postgresql+psycopg2"
}
},
@ -37,15 +41,21 @@
"description": "Username to connect to Postgres. This user should have privileges to read all the metadata in Postgres.",
"type": "string"
},
"password": {
"title": "Password",
"description": "Password to connect to Postgres.",
"type": "string",
"format": "password"
"authType": {
"title": "Auth Configuration Type",
"description": "Choose Auth Config Type.",
"oneOf": [
{
"$ref": "./common/basicAuth.json"
},
{
"$ref": "./common/iamAuthConfig.json"
}
]
},
"hostPort": {
"title": "Host and Port",
"description": "Host and port of the Postgres service.",
"description": "Host and port of the source service.",
"type": "string"
},
"database": {
@ -56,7 +66,14 @@
"sslMode": {
"title": "SSL Mode",
"description": "SSL Mode to connect to postgres database.",
"enum": ["disable", "allow", "prefer", "require", "verify-ca", "verify-full"],
"enum": [
"disable",
"allow",
"prefer",
"require",
"verify-ca",
"verify-full"
],
"default": "disable"
},
"sslConfig": {
@ -109,5 +126,9 @@
}
},
"additionalProperties": false,
"required": ["hostPort", "username", "database"]
"required": [
"hostPort",
"username",
"database"
]
}

2
openmetadata-ui/src/main/resources/ui/cypress/common/common.js
View File

@ -1045,7 +1045,7 @@ export const updateOwner = () => {
export const mySqlConnectionInput = () => {
cy.get('#root\\/username').type(Cypress.env('mysqlUsername'));
checkServiceFieldSectionHighlighting('username');
cy.get('#root\\/password').type(Cypress.env('mysqlPassword'));
cy.get('#root\\/authType\\/password').type(Cypress.env('mysqlPassword'));
checkServiceFieldSectionHighlighting('password');
cy.get('#root\\/hostPort').type(Cypress.env('mysqlHostPort'));
checkServiceFieldSectionHighlighting('hostPort');

2
openmetadata-ui/src/main/resources/ui/cypress/e2e/AddNewService/postgres.spec.js
View File

@ -53,7 +53,7 @@ describe('Postgres Ingestion', () => {
.scrollIntoView()
.type(Cypress.env('postgresUsername'));
checkServiceFieldSectionHighlighting('username');
cy.get('#root\\/password')
cy.get('#root\\/authType\\/password')
.scrollIntoView()
.type(Cypress.env('postgresPassword'));
checkServiceFieldSectionHighlighting('password');

93
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Mysql.md
View File

@ -41,11 +41,103 @@ $$section
Username to connect to MySQL. This user should have access to the `INFORMATION_SCHEMA` to extract metadata. Other workflows may require different permissions -- refer to the section above for more information.
$$
## Basic Auth
$$section
### Password $(id="password")
Password to connect to MySQL.
$$
## IAM Auth Config
### Note: If you are using IAM auth, add `"ssl": {"ssl-mode": "allow"} under Connection Arguments`
$$section
### AWS Access Key ID $(id="awsAccessKeyId")
When you interact with AWS, you specify your AWS security credentials to verify who you are and whether you have permission to access the resources that you are requesting. AWS uses the security credentials to authenticate and authorize your requests ([docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html)).
Access keys consist of two parts:
1. An access key ID (for example, `AKIAIOSFODNN7EXAMPLE`),
2. And a secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
You must use both the access key ID and secret access key together to authenticate your requests.
You can find further information on how to manage your access keys [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html)
$$
$$section
### AWS Secret Access Key $(id="awsSecretAccessKey")
Secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
$$
$$section
### AWS Region $(id="awsRegion")
Each AWS Region is a separate geographic area in which AWS clusters data centers ([docs](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html)).
As AWS can have instances in multiple regions, we need to know the region the service you want reach belongs to.
Note that the AWS Region is the only required parameter when configuring a connection. When connecting to the services programmatically, there are different ways in which we can extract and use the rest of AWS configurations. You can find further information about configuring your credentials [here](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials).
$$
$$section
### AWS Session Token $(id="awsSessionToken")
If you are using temporary credentials to access your services, you will need to inform the AWS Access Key ID and AWS Secrets Access Key. Also, these will include an AWS Session Token.
You can find more information on [Using temporary credentials with AWS resources](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html).
$$
$$section
### Endpoint URL $(id="endPointURL")
To connect programmatically to an AWS service, you use an endpoint. An *endpoint* is the URL of the entry point for an AWS web service. The AWS SDKs and the AWS Command Line Interface (AWS CLI) automatically use the default endpoint for each service in an AWS Region. But you can specify an alternate endpoint for your API requests.
Find more information on [AWS service endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html).
$$
$$section
### Profile Name $(id="profileName")
A named profile is a collection of settings and credentials that you can apply to an AWS CLI command. When you specify a profile to run a command, the settings and credentials are used to run that command. Multiple named profiles can be stored in the config and credentials files.
You can inform this field if you'd like to use a profile other than `default`.
Find here more information about [Named profiles for the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html).
$$
$$section
### Assume Role ARN $(id="assumeRoleArn")
Typically, you use `AssumeRole` within your account or for cross-account access. In this field you'll set the `ARN` (Amazon Resource Name) of the policy of the other account.
A user who wants to access a role in a different account must also have permissions that are delegated from the account administrator. The administrator must attach a policy that allows the user to call `AssumeRole` for the `ARN` of the role in the other account.
This is a required field if you'd like to `AssumeRole`.
Find more information on [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html).
$$
$$section
### Assume Role Session Name $(id="assumeRoleSessionName")
An identifier for the assumed role session. Use the role session name to uniquely identify a session when the same role is assumed by different principals or for different reasons.
By default, we'll use the name `OpenMetadataSession`.
Find more information about the [Role Session Name](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=An%20identifier%20for%20the%20assumed%20role%20session.).
$$
$$section
### Assume Role Source Identity $(id="assumeRoleSourceIdentity")
The source identity specified by the principal that is calling the `AssumeRole` operation. You can use source identity information in AWS CloudTrail logs to determine who took actions with a role.
Find more information about [Source Identity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=Required%3A%20No-,SourceIdentity,-The%20source%20identity).
$$
$$section
### Host Port $(id="hostPort")
Host and port of the MySQL service. This should be specified as a string in the format `hostname:port`. E.g., `localhost:3306`, `host.docker.internal:3306`.
@ -88,4 +180,5 @@ $$
$$section
### Connection Arguments $(id="connectionArguments")
Additional connection arguments such as security or protocol configs that can be sent to the service during connection.
#### Note: If you are using IAM auth, add `"ssl": {"ssl-mode": "allow"} under Connection Arguments`
$$

91
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Postgres.md
View File

@ -45,13 +45,101 @@ $$section
Username to connect to Postgres. This user should have privileges to read all the metadata in Postgres.
$$
## Basic Auth
$$section
### Password $(id="password")
Password to connect to Postgres.
$$
## IAM Auth Config
$$section
### AWS Access Key ID $(id="awsAccessKeyId")
When you interact with AWS, you specify your AWS security credentials to verify who you are and whether you have permission to access the resources that you are requesting. AWS uses the security credentials to authenticate and authorize your requests ([docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html)).
Access keys consist of two parts:
1. An access key ID (for example, `AKIAIOSFODNN7EXAMPLE`),
2. And a secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
You must use both the access key ID and secret access key together to authenticate your requests.
You can find further information on how to manage your access keys [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html)
$$
$$section
### AWS Secret Access Key $(id="awsSecretAccessKey")
Secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
$$
$$section
### AWS Region $(id="awsRegion")
Each AWS Region is a separate geographic area in which AWS clusters data centers ([docs](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html)).
As AWS can have instances in multiple regions, we need to know the region the service you want reach belongs to.
Note that the AWS Region is the only required parameter when configuring a connection. When connecting to the services programmatically, there are different ways in which we can extract and use the rest of AWS configurations. You can find further information about configuring your credentials [here](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials).
$$
$$section
### AWS Session Token $(id="awsSessionToken")
If you are using temporary credentials to access your services, you will need to inform the AWS Access Key ID and AWS Secrets Access Key. Also, these will include an AWS Session Token.
You can find more information on [Using temporary credentials with AWS resources](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html).
$$
$$section
### Endpoint URL $(id="endPointURL")
To connect programmatically to an AWS service, you use an endpoint. An *endpoint* is the URL of the entry point for an AWS web service. The AWS SDKs and the AWS Command Line Interface (AWS CLI) automatically use the default endpoint for each service in an AWS Region. But you can specify an alternate endpoint for your API requests.
Find more information on [AWS service endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html).
$$
$$section
### Profile Name $(id="profileName")
A named profile is a collection of settings and credentials that you can apply to an AWS CLI command. When you specify a profile to run a command, the settings and credentials are used to run that command. Multiple named profiles can be stored in the config and credentials files.
You can inform this field if you'd like to use a profile other than `default`.
Find here more information about [Named profiles for the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html).
$$
$$section
### Assume Role ARN $(id="assumeRoleArn")
Typically, you use `AssumeRole` within your account or for cross-account access. In this field you'll set the `ARN` (Amazon Resource Name) of the policy of the other account.
A user who wants to access a role in a different account must also have permissions that are delegated from the account administrator. The administrator must attach a policy that allows the user to call `AssumeRole` for the `ARN` of the role in the other account.
This is a required field if you'd like to `AssumeRole`.
Find more information on [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html).
$$
$$section
### Assume Role Session Name $(id="assumeRoleSessionName")
An identifier for the assumed role session. Use the role session name to uniquely identify a session when the same role is assumed by different principals or for different reasons.
By default, we'll use the name `OpenMetadataSession`.
Find more information about the [Role Session Name](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=An%20identifier%20for%20the%20assumed%20role%20session.).
$$
$$section
### Assume Role Source Identity $(id="assumeRoleSourceIdentity")
The source identity specified by the principal that is calling the `AssumeRole` operation. You can use source identity information in AWS CloudTrail logs to determine who took actions with a role.
Find more information about [Source Identity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=Required%3A%20No-,SourceIdentity,-The%20source%20identity).
$$
$$section
### Host and Port $(id="hostPort")
@ -68,6 +156,7 @@ $$section
### SSL Mode $(id="sslMode")
SSL Mode to connect to postgres database. E.g, `prefer`, `verify-ca` etc.
#### Note: if you are using `IAM auth`, select either `allow` (recommended) or other option based on your use case.
$$
$$section