yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-09 23:40:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Py tests (#4122)

Browse Source 
Fix Py tests (#4122)
This commit is contained in:
Pere Miquel Brull 2022-04-14 11:22:39 +02:00 committed by GitHub
parent 735cc54614
commit 6768bf708d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 78 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json
View File

@ -9,8 +9,8 @@
"clickhouseType": { "clickhouseType": {
"description": "Service type.", "description": "Service type.",
"type": "string", "type": "string",
"enum": ["ClickHouse"], "enum": ["Clickhouse"],
"default": "ClickHouse" "default": "Clickhouse"
}, },
"clickhouseScheme": { "clickhouseScheme": {
"description": "SQLAlchemy driver scheme options.", "description": "SQLAlchemy driver scheme options.",
@ -23,7 +23,7 @@
"type": { "type": {
"description": "Service Type", "description": "Service Type",
"$ref": "#/definitions/clickhouseType", "$ref": "#/definitions/clickhouseType",
"default": "ClickHouse" "default": "Clickhouse"
}, },
"scheme": { "scheme": {
"description": "SQLAlchemy driver scheme options.", "description": "SQLAlchemy driver scheme options.",

14
catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json
View File

@ -9,21 +9,21 @@
"mssqlType": { "mssqlType": {
"description": "Service type.", "description": "Service type.",
"type": "string", "type": "string",
"enum": ["MSSQL"], "enum": ["Mssql"],
"default": "MSSQL" "default": "Mssql"
}, },
"mssqlScheme": { "mssqlScheme": {
"description": "SQLAlchemy driver scheme options.", "description": "SQLAlchemy driver scheme options.",
"type": "string", "type": "string",
"enum": ["mssql+pyodbc", "mssql+pytds", "mssql+pymssql"], "enum": ["mssql+pyodbc", "mssql+pytds", "mssql+pymssql"],
"default": "mysql+pymysql" "default": "mssql+pytds"
} }
}, },
"properties": { "properties": {
"type": { "type": {
"description": "Service Type", "description": "Service Type",
"$ref": "#/definitions/mssqlType", "$ref": "#/definitions/mssqlType",
"default": "MSSQL" "default": "Mssql"
}, },
"scheme": { "scheme": {
"description": "SQLAlchemy driver scheme options.", "description": "SQLAlchemy driver scheme options.",
@ -61,5 +61,9 @@
"$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
} }
}, },
"additionalProperties": false "additionalProperties": false,
"required": [
"hostPort",
"username"
]
} }

8
catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json
View File

@ -14,7 +14,7 @@
"Redshift", "Redshift",
"Snowflake", "Snowflake",
"Postgres", "Postgres",
"MSSQL", "Mssql",
"Hive", "Hive",
"Oracle", "Oracle",
"Athena", "Athena",
@ -25,7 +25,7 @@
"MariaDB", "MariaDB",
"Druid", "Druid",
"Db2", "Db2",
"ClickHouse", "Clickhouse",
"Databricks", "Databricks",
"DynamoDB", "DynamoDB",
"AzureSQL", "AzureSQL",
@ -52,7 +52,7 @@
"name": "Postgres" "name": "Postgres"
}, },
{ {
"name": "MSSQL" "name": "Mssql"
}, },
{ {
"name": "Oracle" "name": "Oracle"
@ -85,7 +85,7 @@
"name": "Db2" "name": "Db2"
}, },
{ {
"name": "ClickHouse" "name": "Clickhouse"
}, },
{ {
"name": "Databricks" "name": "Databricks"

2
ingestion/src/metadata/ingestion/source/amundsen.py
View File

@ -292,7 +292,7 @@ class AmundsenSource(Source[Entity]):
"name": service_name, "name": service_name,
"description": "", "description": "",
"serviceType": self.database_service_map.get( "serviceType": self.database_service_map.get(
service_type.lower(), DatabaseServiceType.MySQL.value service_type.lower(), DatabaseServiceType.Mysql.value
), ),
"connection": {"config": {}}, "connection": {"config": {}},
} }

2
ingestion/src/metadata/ingestion/source/mssql_usage.py
View File

@ -64,7 +64,7 @@ class MssqlUsageSource(Source[TableQuery]):
self.alchemy_helper = SQLAlchemyHelper( self.alchemy_helper = SQLAlchemyHelper(
self.connection, self.connection,
metadata_config, metadata_config,
DatabaseServiceType.MSSQL.value, DatabaseServiceType.Mssql.value,
self.sql_stmt, self.sql_stmt,
) )
self.report = SQLSourceStatus() self.report = SQLSourceStatus()

1
ingestion/src/metadata/ingestion/source/postgres.py
View File

@ -17,7 +17,6 @@ import psycopg2
from sqlalchemy.engine.reflection import Inspector from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.inspection import inspect from sqlalchemy.inspection import inspect
from metadata.config.common import FQDN_SEPARATOR
from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.services.connections.database.postgresConnection import ( from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
PostgresConnection, PostgresConnection,

2
ingestion/tests/integration/lineage/airflow/test_airflow_lineage.py
View File

@ -64,7 +64,7 @@ class AirflowLineageTest(TestCase):
service = CreateDatabaseServiceRequest( service = CreateDatabaseServiceRequest(
name="test-service-table-lineage", name="test-service-table-lineage",
serviceType=DatabaseServiceType.MySQL, serviceType=DatabaseServiceType.Mysql,
connection=DatabaseConnection( connection=DatabaseConnection(
config=MysqlConnection( config=MysqlConnection(
username="username", username="username",

2
ingestion/tests/integration/ometa/test_ometa_database_api.py
View File

@ -56,7 +56,7 @@ class OMetaDatabaseTest(TestCase):
service = CreateDatabaseServiceRequest( service = CreateDatabaseServiceRequest(
name="test-service-db", name="test-service-db",
serviceType=DatabaseServiceType.MySQL, serviceType=DatabaseServiceType.Mysql,
connection=DatabaseConnection( connection=DatabaseConnection(
config=MysqlConnection( config=MysqlConnection(
username="username", username="username",

6
ingestion/tests/integration/ometa/test_ometa_database_service_api.py
View File

@ -58,14 +58,14 @@ class OMetaDatabaseServiceTest(TestCase):
cls.entity = DatabaseService( cls.entity = DatabaseService(
id=uuid.uuid4(), id=uuid.uuid4(),
name="test-db-service", name="test-db-service",
serviceType=DatabaseServiceType.MySQL, serviceType=DatabaseServiceType.Mysql,
connection=cls.connection, connection=cls.connection,
href="http://resource-uri/", # Dummy value, this is auto-generated by OM href="http://resource-uri/", # Dummy value, this is auto-generated by OM
) )
cls.create = CreateDatabaseServiceRequest( cls.create = CreateDatabaseServiceRequest(
name="test-db-service", name="test-db-service",
serviceType=DatabaseServiceType.MySQL, serviceType=DatabaseServiceType.Mysql,
connection=cls.connection, connection=cls.connection,
) )
@ -117,7 +117,7 @@ class OMetaDatabaseServiceTest(TestCase):
update_request = CreateDatabaseServiceRequest( update_request = CreateDatabaseServiceRequest(
name="test-db-service", name="test-db-service",
serviceType=DatabaseServiceType.MySQL, serviceType=DatabaseServiceType.Mysql,
connection=new_connection, connection=new_connection,
) )

2
ingestion/tests/integration/ometa/test_ometa_lineage_api.py
View File

@ -63,7 +63,7 @@ class OMetaLineageTest(TestCase):
db_service = CreateDatabaseServiceRequest( db_service = CreateDatabaseServiceRequest(
name="test-service-db-lineage", name="test-service-db-lineage",
serviceType=DatabaseServiceType.MySQL, serviceType=DatabaseServiceType.Mysql,
connection=DatabaseConnection( connection=DatabaseConnection(
config=MysqlConnection( config=MysqlConnection(
username="username", username="username",

2
ingestion/tests/integration/ometa/test_ometa_model_api.py
View File

@ -199,7 +199,7 @@ class OMetaModelTest(TestCase):
service = CreateDatabaseServiceRequest( service = CreateDatabaseServiceRequest(
name="test-service-table-ml", name="test-service-table-ml",
serviceType=DatabaseServiceType.MySQL, serviceType=DatabaseServiceType.Mysql,
connection=DatabaseConnection( connection=DatabaseConnection(
config=MysqlConnection( config=MysqlConnection(
username="username", username="username",

10
ingestion/tests/integration/ometa/test_ometa_service_api.py
View File

@ -57,7 +57,7 @@ class OMetaServiceTest(TestCase):
"serviceName": "local_mysql", "serviceName": "local_mysql",
"serviceConnection": { "serviceConnection": {
"config": { "config": {
"type": "MySQL", "type": "Mysql",
"username": "openmetadata_user", "username": "openmetadata_user",
"password": "openmetadata_password", "password": "openmetadata_password",
"hostPort": "random:3306", "hostPort": "random:3306",
@ -73,7 +73,7 @@ class OMetaServiceTest(TestCase):
entity=DatabaseService, config=workflow_source entity=DatabaseService, config=workflow_source
) )
assert service assert service
assert service.serviceType == DatabaseServiceType.MySQL assert service.serviceType == DatabaseServiceType.Mysql
# Check get # Check get
assert service == self.metadata.get_service_or_create( assert service == self.metadata.get_service_or_create(
@ -92,10 +92,10 @@ class OMetaServiceTest(TestCase):
"serviceName": "local_mssql", "serviceName": "local_mssql",
"serviceConnection": { "serviceConnection": {
"config": { "config": {
"type": "MSSQL", "type": "Mssql",
"username": "openmetadata_user", "username": "openmetadata_user",
"password": "openmetadata_password", "password": "openmetadata_password",
"hostPort": "random:3306", "hostPort": "random:1433",
} }
}, },
"sourceConfig": {"config": {"enableDataProfiler": False}}, "sourceConfig": {"config": {"enableDataProfiler": False}},
@ -108,7 +108,7 @@ class OMetaServiceTest(TestCase):
entity=DatabaseService, config=workflow_source entity=DatabaseService, config=workflow_source
) )
assert service assert service
assert service.serviceType == DatabaseServiceType.MSSQL assert service.serviceType == DatabaseServiceType.Mssql
# Check get # Check get
assert service == self.metadata.get_service_or_create( assert service == self.metadata.get_service_or_create(

2
ingestion/tests/integration/ometa/test_ometa_table_api.py
View File

@ -86,7 +86,7 @@ class OMetaTableTest(TestCase):
service = CreateDatabaseServiceRequest( service = CreateDatabaseServiceRequest(
name="test-service-table", name="test-service-table",
serviceType=DatabaseServiceType.MySQL, serviceType=DatabaseServiceType.Mysql,
connection=DatabaseConnection( connection=DatabaseConnection(
config=MysqlConnection( config=MysqlConnection(
username="username", username="username",

9
ingestion/tests/unit/source/test_bigquery.py
View File

@ -262,16 +262,13 @@ class BigQueryIngestionTest(TestCase):
execute_workflow(json.loads(CONFIG)) execute_workflow(json.loads(CONFIG))
def test_file_sink(self):
config = json.loads(CONFIG) config = json.loads(CONFIG)
file_data = open(config["sink"]["config"]["filename"]) file_data = open(config["sink"]["config"]["filename"])
file_sink = json.load(file_data) file_sink = json.load(file_data)
for ometa_data in file_sink: for ometa_data in file_sink:
table = ometa_data.get("table") table = ometa_data.get("table")
omdtable_obj: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj( _: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(ometa_data)
ometa_data _: Table = Table.parse_obj(table)
)
table_obj: Table = Table.parse_obj(table)
assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text") assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text")
table_name = ( table_name = (
@ -281,7 +278,7 @@ class BigQueryIngestionTest(TestCase):
assert table_name in MOCK_GET_TABLE_NAMES assert table_name in MOCK_GET_TABLE_NAMES
for column in table.get("columns"): for column in table.get("columns"):
column_obj: Column = Column.parse_obj(column) _: Column = Column.parse_obj(column)
if column in MOCK_UNIQUE_CONSTRAINTS: if column in MOCK_UNIQUE_CONSTRAINTS:
assert Column.constraint.UNIQUE == column.get("constraint") assert Column.constraint.UNIQUE == column.get("constraint")
if column in MOCK_PK_CONSTRAINT.get("constrained_columns"): if column in MOCK_PK_CONSTRAINT.get("constrained_columns"):

13
ingestion/tests/unit/source/test_clickhouse.py
View File

@ -39,9 +39,9 @@ CONFIG = """
"serviceName": "local_clickhouse", "serviceName": "local_clickhouse",
"serviceConnection": { "serviceConnection": {
"config": { "config": {
"type": "ClickHouse", "type": "Clickhouse",
"username":"default", "username": "default",
"password":"", "password": "",
"hostPort": "localhost:8123", "hostPort": "localhost:8123",
"database": "default" "database": "default"
} }
@ -213,14 +213,13 @@ class ClickhouseIngestionTest(TestCase):
execute_workflow() execute_workflow()
def test_file_sink(self):
config = json.loads(CONFIG) config = json.loads(CONFIG)
file_data = open(config["sink"]["config"]["filename"]) file_data = open(config["sink"]["config"]["filename"])
data = json.load(file_data) data = json.load(file_data)
for i in data: for i in data:
table = i.get("table") table = i.get("table")
omdtable_obj: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) _: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i)
table_obj: Table = Table.parse_obj(table) _: Table = Table.parse_obj(table)
assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text") assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text")
@ -228,7 +227,7 @@ class ClickhouseIngestionTest(TestCase):
assert table.get("name") in MOCK_GET_TABLE_NAMES assert table.get("name") in MOCK_GET_TABLE_NAMES
for column in table.get("columns"): for column in table.get("columns"):
column_obj: Column = Column.parse_obj(column) _: Column = Column.parse_obj(column)
if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"): if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"):
assert Column.constraint.UNIQUE == column.get("constraint") assert Column.constraint.UNIQUE == column.get("constraint")
if column in MOCK_PK_CONSTRAINT.get("constrained_columns"): if column in MOCK_PK_CONSTRAINT.get("constrained_columns"):

67
ingestion/tests/unit/source/test_mysql.py
View File

@ -35,39 +35,43 @@ from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
CONFIG = """ CONFIG = """
{ {
"source": { "source": {
"type": "mysql", "type": "mysql",
"serviceName": "local_mysql", "serviceName": "local_mysql",
"serviceConnection": { "serviceConnection": {
"config": { "config": {
"type": "MySQL", "type": "Mysql",
"username": "openmetadata_user", "username": "openmetadata_user",
"password": "openmetadata_password", "password": "openmetadata_password",
"hostPort": "localhost:3306", "hostPort": "localhost:3306",
"database": "openmetadata_db" "database": "openmetadata_db"
} }
},
"sourceConfig": {
"config": {
"enableDataProfiler": false,
"schemaFilterPattern": {
"excludes": [
"system.*",
"information_schema.*",
"INFORMATION_SCHEMA.*"
]
}
}
}
}, },
"sourceConfig": { "sink": {
"type": "file",
"config": { "config": {
"enableDataProfiler": false, "filename": "/var/tmp/datasets.json"
"schemaFilterPattern":{
"excludes": ["system.*","information_schema.*","INFORMATION_SCHEMA.*"]
} }
}
}
}, },
"sink": { "workflowConfig": {
"type": "file", "openMetadataServerConfig": {
"config": { "hostPort": "http://localhost:8585/api",
"filename": "/var/tmp/datasets.json" "authProvider": "no-auth"
}
} }
},
"workflowConfig": {
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "no-auth"
}
}
} }
""" """
@ -278,14 +282,13 @@ class MySqlIngestionTest(TestCase):
execute_workflow() execute_workflow()
def test_file_sink(self):
config = json.loads(CONFIG) config = json.loads(CONFIG)
file_data = open(config["sink"]["config"]["filename"]) file_data = open(config["sink"]["config"]["filename"])
data = json.load(file_data) data = json.load(file_data)
for i in data: for i in data:
table = i.get("table") table = i.get("table")
omdtable_obj: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) _: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i)
table_obj: Table = Table.parse_obj(table) _: Table = Table.parse_obj(table)
assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text") assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text")
@ -293,7 +296,7 @@ class MySqlIngestionTest(TestCase):
assert table.get("name") in MOCK_GET_TABLE_NAMES assert table.get("name") in MOCK_GET_TABLE_NAMES
for column in table.get("columns"): for column in table.get("columns"):
column_obj: Column = Column.parse_obj(column) _: Column = Column.parse_obj(column)
if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"): if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"):
assert Column.constraint.UNIQUE == column.get("constraint") assert Column.constraint.UNIQUE == column.get("constraint")
if column in MOCK_PK_CONSTRAINT.get("constrained_columns"): if column in MOCK_PK_CONSTRAINT.get("constrained_columns"):

7
ingestion/tests/unit/source/test_vertica.py
View File

@ -215,14 +215,13 @@ class VerticaIngestionTest(TestCase):
get_view_definition.return_value = MOCK_GET_VIEW_DEFINITION get_view_definition.return_value = MOCK_GET_VIEW_DEFINITION
execute_workflow() execute_workflow()
def test_file_sink(self):
config = json.loads(CONFIG) config = json.loads(CONFIG)
file_data = open(config["sink"]["config"]["filename"]) file_data = open(config["sink"]["config"]["filename"])
data = json.load(file_data) data = json.load(file_data)
for i in data: for i in data:
table = i.get("table") table = i.get("table")
omdtable_obj: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i) _: OMetaDatabaseAndTable = OMetaDatabaseAndTable.parse_obj(i)
table_obj: Table = Table.parse_obj(table) _: Table = Table.parse_obj(table)
assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text") assert table.get("description") == GET_TABLE_DESCRIPTIONS.get("text")
@ -230,7 +229,7 @@ class VerticaIngestionTest(TestCase):
assert table.get("name") in MOCK_GET_TABLE_NAMES assert table.get("name") in MOCK_GET_TABLE_NAMES
for column in table.get("columns"): for column in table.get("columns"):
column_obj: Column = Column.parse_obj(column) _: Column = Column.parse_obj(column)
if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"): if column in MOCK_UNIQUE_CONSTRAINTS[0].get("column_names"):
assert Column.constraint.UNIQUE == column.get("constraint") assert Column.constraint.UNIQUE == column.get("constraint")
if column in MOCK_PK_CONSTRAINT.get("constrained_columns"): if column in MOCK_PK_CONSTRAINT.get("constrained_columns"):